diff --git "a/8b7178b178b/3430821.out" "b/8b7178b178b/3430821.out" new file mode 100644--- /dev/null +++ "b/8b7178b178b/3430821.out" @@ -0,0 +1,51797 @@ +Model parameters: d_model 4096 ffw_size 16384 kv_size 128 n_heads 32 n_layers 42 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 2 --num-layers 42 --hidden-size 4096 --num-attention-heads 32 --kv-channels 128 --ffn-hidden-size 16384 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 1 --global-batch-size 512 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-8b7178b178bval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 5000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_8b7178b178bval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save lm1-8b7-178b-c4-repetitions/8b7178b178b --load lm1-8b7-178b-c4-repetitions/8b7178b178b --train-weighted-split-paths-path train400m.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --num-workers 0 --valid-num-workers 0 --deepspeed --deepspeed_config ds_configs/3430821.json --zero-stage 0 +START 3430821: Sat 29 Apr 2023 09:42:33 AM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 46.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 38.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== +61: +61: +61: ======================= ROCm System Management Interface ======================= +61: ================================= Concise Info ================================= +61: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +61: 0 47.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +61: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +61: 2 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +61: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +61: 4 48.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +61: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +61: 6 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +61: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +61: ================================================================================ +61: ============================= End of ROCm SMI Log ============================== +54: +54: +54: ======================= ROCm System Management Interface ======================= +54: ================================= Concise Info ================================= +54: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +54: 0 48.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +54: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +54: 2 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +54: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +54: 4 49.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +54: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +54: 6 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +54: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +54: ================================================================================ +54: ============================= End of ROCm SMI Log ============================== +63: +63: +63: ======================= ROCm System Management Interface ======================= +63: ================================= Concise Info ================================= +63: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +63: 0 49.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +63: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +63: 2 42.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +63: 3 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +63: 4 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +63: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +63: 6 35.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +63: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +63: ================================================================================ +63: ============================= End of ROCm SMI Log ============================== +59: +59: +59: ======================= ROCm System Management Interface ======================= +59: ================================= Concise Info ================================= +59: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +59: 0 48.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +59: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +59: 2 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +59: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +59: 4 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +59: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +59: 6 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +59: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +59: ================================================================================ +59: ============================= End of ROCm SMI Log ============================== +58: +58: +58: ======================= ROCm System Management Interface ======================= +58: ================================= Concise Info ================================= +58: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +58: 0 54.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +58: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +58: 2 41.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +58: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +58: 4 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +58: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +58: 6 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +58: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +58: ================================================================================ +58: ============================= End of ROCm SMI Log ============================== +57: +57: +57: ======================= ROCm System Management Interface ======================= +57: ================================= Concise Info ================================= +57: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +57: 0 49.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +57: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +57: 2 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +57: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +57: 4 52.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +57: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +57: 6 39.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +57: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +57: ================================================================================ +57: ============================= End of ROCm SMI Log ============================== +60: +60: +60: ======================= ROCm System Management Interface ======================= +60: ================================= Concise Info ================================= +60: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +60: 0 41.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +60: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +60: 2 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +60: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +60: 4 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +60: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +60: 6 48.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +60: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +60: ================================================================================ +60: ============================= End of ROCm SMI Log ============================== +56: +56: +56: ======================= ROCm System Management Interface ======================= +56: ================================= Concise Info ================================= +56: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +56: 0 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +56: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +56: 2 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +56: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +56: 4 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +56: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +56: 6 44.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +56: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +56: ================================================================================ +56: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 47.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 42.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 46.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 36.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +44: +44: +44: ======================= ROCm System Management Interface ======================= +44: ================================= Concise Info ================================= +44: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +44: 0 46.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +44: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +44: 2 37.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +44: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +44: 4 48.0c 79.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +44: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +44: 6 40.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +44: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +44: ================================================================================ +44: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 49.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 36.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 37.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 36.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 41.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +37: +37: +37: ======================= ROCm System Management Interface ======================= +37: ================================= Concise Info ================================= +37: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +37: 0 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +37: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +37: 2 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +37: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +37: 4 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +37: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +37: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +37: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +37: ================================================================================ +37: ============================= End of ROCm SMI Log ============================== +41: +41: +41: ======================= ROCm System Management Interface ======================= +41: ================================= Concise Info ================================= +41: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +41: 0 48.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +41: 1 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +41: 2 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +41: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +41: 4 51.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +41: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +41: 6 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +41: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +41: ================================================================================ +41: ============================= End of ROCm SMI Log ============================== +53: +53: +53: ======================= ROCm System Management Interface ======================= +53: ================================= Concise Info ================================= +53: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +53: 0 48.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +53: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +53: 2 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +53: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +53: 4 42.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +53: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +53: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +53: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +53: ================================================================================ +53: ============================= End of ROCm SMI Log ============================== +35: +35: +35: ======================= ROCm System Management Interface ======================= +35: ================================= Concise Info ================================= +35: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +35: 0 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +35: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +35: 2 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +35: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +35: 4 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +35: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +35: 6 44.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +35: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +35: ================================================================================ +35: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 43.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 39.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== +51: +51: +51: ======================= ROCm System Management Interface ======================= +51: ================================= Concise Info ================================= +51: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +51: 0 49.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +51: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +51: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +51: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +51: 4 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +51: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +51: 6 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +51: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +51: ================================================================================ +51: ============================= End of ROCm SMI Log ============================== +55: +55: +55: ======================= ROCm System Management Interface ======================= +55: ================================= Concise Info ================================= +55: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +55: 0 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +55: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +55: 2 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +55: 3 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +55: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +55: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +55: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +55: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +55: ================================================================================ +55: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 39.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 40.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 46.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 38.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 42.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 44.0c 80.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 47.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 40.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 37.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 44.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 43.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 42.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== +34: +34: +34: ======================= ROCm System Management Interface ======================= +34: ================================= Concise Info ================================= +34: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +34: 0 51.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +34: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +34: 2 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +34: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +34: 4 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +34: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +34: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +34: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +34: ================================================================================ +34: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 48.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 38.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 39.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 50.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +43: +43: +43: ======================= ROCm System Management Interface ======================= +43: ================================= Concise Info ================================= +43: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +43: 0 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +43: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +43: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +43: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +43: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +43: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +43: 6 38.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +43: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +43: ================================================================================ +43: ============================= End of ROCm SMI Log ============================== +48: +48: +48: ======================= ROCm System Management Interface ======================= +48: ================================= Concise Info ================================= +48: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +48: 0 48.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +48: 1 53.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +48: 2 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +48: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +48: 4 49.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +48: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +48: 6 39.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +48: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +48: ================================================================================ +48: ============================= End of ROCm SMI Log ============================== +47: +47: +47: ======================= ROCm System Management Interface ======================= +47: ================================= Concise Info ================================= +47: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +47: 0 50.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +47: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +47: 2 44.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +47: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +47: 4 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +47: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +47: 6 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +47: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +47: ================================================================================ +47: ============================= End of ROCm SMI Log ============================== +42: +42: +42: ======================= ROCm System Management Interface ======================= +42: ================================= Concise Info ================================= +42: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +42: 0 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +42: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +42: 2 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +42: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +42: 4 47.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +42: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +42: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +42: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +42: ================================================================================ +42: ============================= End of ROCm SMI Log ============================== +33: +33: +33: ======================= ROCm System Management Interface ======================= +33: ================================= Concise Info ================================= +33: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +33: 0 52.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +33: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +33: 2 39.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +33: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +33: 4 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +33: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +33: 6 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +33: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +33: ================================================================================ +33: ============================= End of ROCm SMI Log ============================== +62: +62: +62: ======================= ROCm System Management Interface ======================= +62: ================================= Concise Info ================================= +62: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +62: 0 53.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +62: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +62: 2 47.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +62: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +62: 4 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +62: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +62: 6 50.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +62: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +62: ================================================================================ +62: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 49.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 43.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 38.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 35.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== +38: +38: +38: ======================= ROCm System Management Interface ======================= +38: ================================= Concise Info ================================= +38: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +38: 0 47.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +38: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +38: 2 40.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +38: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +38: 4 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +38: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +38: 6 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +38: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +38: ================================================================================ +38: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 54.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 39.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== +46: +46: +46: ======================= ROCm System Management Interface ======================= +46: ================================= Concise Info ================================= +46: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +46: 0 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +46: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +46: 2 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +46: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +46: 4 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +46: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +46: 6 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +46: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +46: ================================================================================ +46: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 48.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 38.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 52.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 34.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 46.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 39.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== +49: +49: +49: ======================= ROCm System Management Interface ======================= +49: ================================= Concise Info ================================= +49: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +49: 0 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +49: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +49: 2 47.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +49: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +49: 4 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +49: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +49: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +49: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +49: ================================================================================ +49: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 41.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 41.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 39.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 37.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 49.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 49.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +39: +39: +39: ======================= ROCm System Management Interface ======================= +39: ================================= Concise Info ================================= +39: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +39: 0 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +39: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +39: 2 51.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +39: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +39: 4 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +39: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +39: 6 47.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +39: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +39: ================================================================================ +39: ============================= End of ROCm SMI Log ============================== +36: +36: +36: ======================= ROCm System Management Interface ======================= +36: ================================= Concise Info ================================= +36: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +36: 0 47.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +36: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +36: 2 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +36: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +36: 4 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +36: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +36: 6 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +36: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +36: ================================================================================ +36: ============================= End of ROCm SMI Log ============================== +32: +32: +32: ======================= ROCm System Management Interface ======================= +32: ================================= Concise Info ================================= +32: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +32: 0 49.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +32: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +32: 2 44.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +32: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +32: 4 52.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +32: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +32: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +32: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +32: ================================================================================ +32: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 37.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 37.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 38.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 47.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== +50: +50: +50: ======================= ROCm System Management Interface ======================= +50: ================================= Concise Info ================================= +50: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +50: 0 51.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +50: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +50: 2 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +50: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +50: 4 44.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +50: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +50: 6 36.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +50: 7 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +50: ================================================================================ +50: ============================= End of ROCm SMI Log ============================== +40: +40: +40: ======================= ROCm System Management Interface ======================= +40: ================================= Concise Info ================================= +40: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +40: 0 45.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +40: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +40: 2 36.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +40: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +40: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +40: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +40: 6 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +40: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +40: ================================================================================ +40: ============================= End of ROCm SMI Log ============================== +45: +45: +45: ======================= ROCm System Management Interface ======================= +45: ================================= Concise Info ================================= +45: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +45: 0 47.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +45: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +45: 2 39.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +45: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +45: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +45: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +45: 6 36.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +45: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +45: ================================================================================ +45: ============================= End of ROCm SMI Log ============================== +52: +52: +52: ======================= ROCm System Management Interface ======================= +52: ================================= Concise Info ================================= +52: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +52: 0 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +52: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +52: 2 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +52: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +52: 4 39.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +52: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +52: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +52: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +52: ================================================================================ +52: ============================= End of ROCm SMI Log ============================== +25: Launching on nid006096 (25/64), master nid006071 port 9999, GPUs 8, CUDA: True +58: Launching on nid006803 (58/64), master nid006071 port 9999, GPUs 8, CUDA: True +29: Launching on nid006774 (29/64), master nid006071 port 9999, GPUs 8, CUDA: True + 2: Launching on nid006073 (2/64), master nid006071 port 9999, GPUs 8, CUDA: True +12: Launching on nid006083 (12/64), master nid006071 port 9999, GPUs 8, CUDA: True +16: Launching on nid006087 (16/64), master nid006071 port 9999, GPUs 8, CUDA: True +56: Launching on nid006801 (56/64), master nid006071 port 9999, GPUs 8, CUDA: True +23: Launching on nid006094 (23/64), master nid006071 port 9999, GPUs 8, CUDA: True +17: Launching on nid006088 (17/64), master nid006071 port 9999, GPUs 8, CUDA: True +27: Launching on nid006772 (27/64), master nid006071 port 9999, GPUs 8, CUDA: True + 1: Launching on nid006072 (1/64), master nid006071 port 9999, GPUs 8, CUDA: True +60: Launching on nid006805 (60/64), master nid006071 port 9999, GPUs 8, CUDA: True +34: Launching on nid006779 (34/64), master nid006071 port 9999, GPUs 8, CUDA: True +13: Launching on nid006084 (13/64), master nid006071 port 9999, GPUs 8, CUDA: True +14: Launching on nid006085 (14/64), master nid006071 port 9999, GPUs 8, CUDA: True + 5: Launching on nid006076 (5/64), master nid006071 port 9999, GPUs 8, CUDA: True +43: Launching on nid006788 (43/64), master nid006071 port 9999, GPUs 8, CUDA: True +62: Launching on nid006807 (62/64), master nid006071 port 9999, GPUs 8, CUDA: True + 0: Launching on nid006071 (0/64), master nid006071 port 9999, GPUs 8, CUDA: True +41: Launching on nid006786 (41/64), master nid006071 port 9999, GPUs 8, CUDA: True +22: Launching on nid006093 (22/64), master nid006071 port 9999, GPUs 8, CUDA: True +33: Launching on nid006778 (33/64), master nid006071 port 9999, GPUs 8, CUDA: True + 9: Launching on nid006080 (9/64), master nid006071 port 9999, GPUs 8, CUDA: True +18: Launching on nid006089 (18/64), master nid006071 port 9999, GPUs 8, CUDA: True +47: Launching on nid006792 (47/64), master nid006071 port 9999, GPUs 8, CUDA: True +37: Launching on nid006782 (37/64), master nid006071 port 9999, GPUs 8, CUDA: True +59: Launching on nid006804 (59/64), master nid006071 port 9999, GPUs 8, CUDA: True +48: Launching on nid006793 (48/64), master nid006071 port 9999, GPUs 8, CUDA: True +44: Launching on nid006789 (44/64), master nid006071 port 9999, GPUs 8, CUDA: True +63: Launching on nid006808 (63/64), master nid006071 port 9999, GPUs 8, CUDA: True +61: Launching on nid006806 (61/64), master nid006071 port 9999, GPUs 8, CUDA: True +26: Launching on nid006097 (26/64), master nid006071 port 9999, GPUs 8, CUDA: True + 3: Launching on nid006074 (3/64), master nid006071 port 9999, GPUs 8, CUDA: True +42: Launching on nid006787 (42/64), master nid006071 port 9999, GPUs 8, CUDA: True +51: Launching on nid006796 (51/64), master nid006071 port 9999, GPUs 8, CUDA: True +55: Launching on nid006800 (55/64), master nid006071 port 9999, GPUs 8, CUDA: True +53: Launching on nid006798 (53/64), master nid006071 port 9999, GPUs 8, CUDA: True +10: Launching on nid006081 (10/64), master nid006071 port 9999, GPUs 8, CUDA: True +30: Launching on nid006775 (30/64), master nid006071 port 9999, GPUs 8, CUDA: True +24: Launching on nid006095 (24/64), master nid006071 port 9999, GPUs 8, CUDA: True +38: Launching on nid006783 (38/64), master nid006071 port 9999, GPUs 8, CUDA: True +15: Launching on nid006086 (15/64), master nid006071 port 9999, GPUs 8, CUDA: True +19: Launching on nid006090 (19/64), master nid006071 port 9999, GPUs 8, CUDA: True +57: Launching on nid006802 (57/64), master nid006071 port 9999, GPUs 8, CUDA: True + 8: Launching on nid006079 (8/64), master nid006071 port 9999, GPUs 8, CUDA: True +28: Launching on nid006773 (28/64), master nid006071 port 9999, GPUs 8, CUDA: True +21: Launching on nid006092 (21/64), master nid006071 port 9999, GPUs 8, CUDA: True +54: Launching on nid006799 (54/64), master nid006071 port 9999, GPUs 8, CUDA: True +35: Launching on nid006780 (35/64), master nid006071 port 9999, GPUs 8, CUDA: True +46: Launching on nid006791 (46/64), master nid006071 port 9999, GPUs 8, CUDA: True +20: Launching on nid006091 (20/64), master nid006071 port 9999, GPUs 8, CUDA: True + 4: Launching on nid006075 (4/64), master nid006071 port 9999, GPUs 8, CUDA: True + 7: Launching on nid006078 (7/64), master nid006071 port 9999, GPUs 8, CUDA: True +49: Launching on nid006794 (49/64), master nid006071 port 9999, GPUs 8, CUDA: True +36: Launching on nid006781 (36/64), master nid006071 port 9999, GPUs 8, CUDA: True +50: Launching on nid006795 (50/64), master nid006071 port 9999, GPUs 8, CUDA: True +32: Launching on nid006777 (32/64), master nid006071 port 9999, GPUs 8, CUDA: True +40: Launching on nid006785 (40/64), master nid006071 port 9999, GPUs 8, CUDA: True +31: Launching on nid006776 (31/64), master nid006071 port 9999, GPUs 8, CUDA: True +11: Launching on nid006082 (11/64), master nid006071 port 9999, GPUs 8, CUDA: True + 6: Launching on nid006077 (6/64), master nid006071 port 9999, GPUs 8, CUDA: True +45: Launching on nid006790 (45/64), master nid006071 port 9999, GPUs 8, CUDA: True +52: Launching on nid006797 (52/64), master nid006071 port 9999, GPUs 8, CUDA: True +39: Launching on nid006784 (39/64), master nid006071 port 9999, GPUs 8, CUDA: True + 0: using world size: 512, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 2 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3430821.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 16384 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 512 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 4096 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-8b7178b178bval + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ lm1-8b7-178b-c4-repetitions/8b7178b178b + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 1 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 32 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 42 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 0 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 2 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ lm1-8b7-178b-c4-repetitions/8b7178b178b + 0: save_interval ................................... 5000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_8b7178b178bval + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_400M_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 0 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 512 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-04-29 09:44:04,749] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +63: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 2 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.094 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 87 + 0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.o scaled_upper_triang_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 63 + 0: ninja: no work to do. + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: ninja: no work to do. + 0: >>> done with compiling and loading fused kernels. Compilation time: 28.651 seconds + 0: time to initialize megatron (seconds): 93.241 + 0: [after megatron is initialized] datetime: 2023-04-29 09:44:45 + 0: building GPT model ... + 0: [2023-04-29 09:44:45,401] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-04-29 09:44:45,402] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-04-29 09:44:45,402] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.63 GB, percent = 7.7% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255, ProcessCoord(pipe=1, data=0, model=0): 256, ProcessCoord(pipe=1, data=0, model=1): 257, ProcessCoord(pipe=1, data=1, model=0): 258, ProcessCoord(pipe=1, data=1, model=1): 259, ProcessCoord(pipe=1, data=2, model=0): 260, ProcessCoord(pipe=1, data=2, model=1): 261, ProcessCoord(pipe=1, data=3, model=0): 262, ProcessCoord(pipe=1, data=3, model=1): 263, ProcessCoord(pipe=1, data=4, model=0): 264, ProcessCoord(pipe=1, data=4, model=1): 265, ProcessCoord(pipe=1, data=5, model=0): 266, ProcessCoord(pipe=1, data=5, model=1): 267, ProcessCoord(pipe=1, data=6, model=0): 268, ProcessCoord(pipe=1, data=6, model=1): 269, ProcessCoord(pipe=1, data=7, model=0): 270, ProcessCoord(pipe=1, data=7, model=1): 271, ProcessCoord(pipe=1, data=8, model=0): 272, ProcessCoord(pipe=1, data=8, model=1): 273, ProcessCoord(pipe=1, data=9, mode + 0: l=0): 274, ProcessCoord(pipe=1, data=9, model=1): 275, ProcessCoord(pipe=1, data=10, model=0): 276, ProcessCoord(pipe=1, data=10, model=1): 277, ProcessCoord(pipe=1, data=11, model=0): 278, ProcessCoord(pipe=1, data=11, model=1): 279, ProcessCoord(pipe=1, data=12, model=0): 280, ProcessCoord(pipe=1, data=12, model=1): 281, ProcessCoord(pipe=1, data=13, model=0): 282, ProcessCoord(pipe=1, data=13, model=1): 283, ProcessCoord(pipe=1, data=14, model=0): 284, ProcessCoord(pipe=1, data=14, model=1): 285, ProcessCoord(pipe=1, data=15, model=0): 286, ProcessCoord(pipe=1, data=15, model=1): 287, ProcessCoord(pipe=1, data=16, model=0): 288, ProcessCoord(pipe=1, data=16, model=1): 289, ProcessCoord(pipe=1, data=17, model=0): 290, ProcessCoord(pipe=1, data=17, model=1): 291, ProcessCoord(pipe=1, data=18, model=0): 292, ProcessCoord(pipe=1, data=18, model=1): 293, ProcessCoord(pipe=1, data=19, model=0): 294, ProcessCoord(pipe=1, data=19, model=1): 295, ProcessCoord(pipe=1, data=20, model=0): 296, ProcessCoord(pipe=1, dat + 0: a=20, model=1): 297, ProcessCoord(pipe=1, data=21, model=0): 298, ProcessCoord(pipe=1, data=21, model=1): 299, ProcessCoord(pipe=1, data=22, model=0): 300, ProcessCoord(pipe=1, data=22, model=1): 301, ProcessCoord(pipe=1, data=23, model=0): 302, ProcessCoord(pipe=1, data=23, model=1): 303, ProcessCoord(pipe=1, data=24, model=0): 304, ProcessCoord(pipe=1, data=24, model=1): 305, ProcessCoord(pipe=1, data=25, model=0): 306, ProcessCoord(pipe=1, data=25, model=1): 307, ProcessCoord(pipe=1, data=26, model=0): 308, ProcessCoord(pipe=1, data=26, model=1): 309, ProcessCoord(pipe=1, data=27, model=0): 310, ProcessCoord(pipe=1, data=27, model=1): 311, ProcessCoord(pipe=1, data=28, model=0): 312, ProcessCoord(pipe=1, data=28, model=1): 313, ProcessCoord(pipe=1, data=29, model=0): 314, ProcessCoord(pipe=1, data=29, model=1): 315, ProcessCoord(pipe=1, data=30, model=0): 316, ProcessCoord(pipe=1, data=30, model=1): 317, ProcessCoord(pipe=1, data=31, model=0): 318, ProcessCoord(pipe=1, data=31, model=1): 319, ProcessCoord( + 0: pipe=1, data=32, model=0): 320, ProcessCoord(pipe=1, data=32, model=1): 321, ProcessCoord(pipe=1, data=33, model=0): 322, ProcessCoord(pipe=1, data=33, model=1): 323, ProcessCoord(pipe=1, data=34, model=0): 324, ProcessCoord(pipe=1, data=34, model=1): 325, ProcessCoord(pipe=1, data=35, model=0): 326, ProcessCoord(pipe=1, data=35, model=1): 327, ProcessCoord(pipe=1, data=36, model=0): 328, ProcessCoord(pipe=1, data=36, model=1): 329, ProcessCoord(pipe=1, data=37, model=0): 330, ProcessCoord(pipe=1, data=37, model=1): 331, ProcessCoord(pipe=1, data=38, model=0): 332, ProcessCoord(pipe=1, data=38, model=1): 333, ProcessCoord(pipe=1, data=39, model=0): 334, ProcessCoord(pipe=1, data=39, model=1): 335, ProcessCoord(pipe=1, data=40, model=0): 336, ProcessCoord(pipe=1, data=40, model=1): 337, ProcessCoord(pipe=1, data=41, model=0): 338, ProcessCoord(pipe=1, data=41, model=1): 339, ProcessCoord(pipe=1, data=42, model=0): 340, ProcessCoord(pipe=1, data=42, model=1): 341, ProcessCoord(pipe=1, data=43, model=0): 342, Pr + 0: ocessCoord(pipe=1, data=43, model=1): 343, ProcessCoord(pipe=1, data=44, model=0): 344, ProcessCoord(pipe=1, data=44, model=1): 345, ProcessCoord(pipe=1, data=45, model=0): 346, ProcessCoord(pipe=1, data=45, model=1): 347, ProcessCoord(pipe=1, data=46, model=0): 348, ProcessCoord(pipe=1, data=46, model=1): 349, ProcessCoord(pipe=1, data=47, model=0): 350, ProcessCoord(pipe=1, data=47, model=1): 351, ProcessCoord(pipe=1, data=48, model=0): 352, ProcessCoord(pipe=1, data=48, model=1): 353, ProcessCoord(pipe=1, data=49, model=0): 354, ProcessCoord(pipe=1, data=49, model=1): 355, ProcessCoord(pipe=1, data=50, model=0): 356, ProcessCoord(pipe=1, data=50, model=1): 357, ProcessCoord(pipe=1, data=51, model=0): 358, ProcessCoord(pipe=1, data=51, model=1): 359, ProcessCoord(pipe=1, data=52, model=0): 360, ProcessCoord(pipe=1, data=52, model=1): 361, ProcessCoord(pipe=1, data=53, model=0): 362, ProcessCoord(pipe=1, data=53, model=1): 363, ProcessCoord(pipe=1, data=54, model=0): 364, ProcessCoord(pipe=1, data=54, model= + 0: 1): 365, ProcessCoord(pipe=1, data=55, model=0): 366, ProcessCoord(pipe=1, data=55, model=1): 367, ProcessCoord(pipe=1, data=56, model=0): 368, ProcessCoord(pipe=1, data=56, model=1): 369, ProcessCoord(pipe=1, data=57, model=0): 370, ProcessCoord(pipe=1, data=57, model=1): 371, ProcessCoord(pipe=1, data=58, model=0): 372, ProcessCoord(pipe=1, data=58, model=1): 373, ProcessCoord(pipe=1, data=59, model=0): 374, ProcessCoord(pipe=1, data=59, model=1): 375, ProcessCoord(pipe=1, data=60, model=0): 376, ProcessCoord(pipe=1, data=60, model=1): 377, ProcessCoord(pipe=1, data=61, model=0): 378, ProcessCoord(pipe=1, data=61, model=1): 379, ProcessCoord(pipe=1, data=62, model=0): 380, ProcessCoord(pipe=1, data=62, model=1): 381, ProcessCoord(pipe=1, data=63, model=0): 382, ProcessCoord(pipe=1, data=63, model=1): 383, ProcessCoord(pipe=1, data=64, model=0): 384, ProcessCoord(pipe=1, data=64, model=1): 385, ProcessCoord(pipe=1, data=65, model=0): 386, ProcessCoord(pipe=1, data=65, model=1): 387, ProcessCoord(pipe=1, data + 0: =66, model=0): 388, ProcessCoord(pipe=1, data=66, model=1): 389, ProcessCoord(pipe=1, data=67, model=0): 390, ProcessCoord(pipe=1, data=67, model=1): 391, ProcessCoord(pipe=1, data=68, model=0): 392, ProcessCoord(pipe=1, data=68, model=1): 393, ProcessCoord(pipe=1, data=69, model=0): 394, ProcessCoord(pipe=1, data=69, model=1): 395, ProcessCoord(pipe=1, data=70, model=0): 396, ProcessCoord(pipe=1, data=70, model=1): 397, ProcessCoord(pipe=1, data=71, model=0): 398, ProcessCoord(pipe=1, data=71, model=1): 399, ProcessCoord(pipe=1, data=72, model=0): 400, ProcessCoord(pipe=1, data=72, model=1): 401, ProcessCoord(pipe=1, data=73, model=0): 402, ProcessCoord(pipe=1, data=73, model=1): 403, ProcessCoord(pipe=1, data=74, model=0): 404, ProcessCoord(pipe=1, data=74, model=1): 405, ProcessCoord(pipe=1, data=75, model=0): 406, ProcessCoord(pipe=1, data=75, model=1): 407, ProcessCoord(pipe=1, data=76, model=0): 408, ProcessCoord(pipe=1, data=76, model=1): 409, ProcessCoord(pipe=1, data=77, model=0): 410, ProcessCoord(p + 0: ipe=1, data=77, model=1): 411, ProcessCoord(pipe=1, data=78, model=0): 412, ProcessCoord(pipe=1, data=78, model=1): 413, ProcessCoord(pipe=1, data=79, model=0): 414, ProcessCoord(pipe=1, data=79, model=1): 415, ProcessCoord(pipe=1, data=80, model=0): 416, ProcessCoord(pipe=1, data=80, model=1): 417, ProcessCoord(pipe=1, data=81, model=0): 418, ProcessCoord(pipe=1, data=81, model=1): 419, ProcessCoord(pipe=1, data=82, model=0): 420, ProcessCoord(pipe=1, data=82, model=1): 421, ProcessCoord(pipe=1, data=83, model=0): 422, ProcessCoord(pipe=1, data=83, model=1): 423, ProcessCoord(pipe=1, data=84, model=0): 424, ProcessCoord(pipe=1, data=84, model=1): 425, ProcessCoord(pipe=1, data=85, model=0): 426, ProcessCoord(pipe=1, data=85, model=1): 427, ProcessCoord(pipe=1, data=86, model=0): 428, ProcessCoord(pipe=1, data=86, model=1): 429, ProcessCoord(pipe=1, data=87, model=0): 430, ProcessCoord(pipe=1, data=87, model=1): 431, ProcessCoord(pipe=1, data=88, model=0): 432, ProcessCoord(pipe=1, data=88, model=1): 433, Pro + 0: cessCoord(pipe=1, data=89, model=0): 434, ProcessCoord(pipe=1, data=89, model=1): 435, ProcessCoord(pipe=1, data=90, model=0): 436, ProcessCoord(pipe=1, data=90, model=1): 437, ProcessCoord(pipe=1, data=91, model=0): 438, ProcessCoord(pipe=1, data=91, model=1): 439, ProcessCoord(pipe=1, data=92, model=0): 440, ProcessCoord(pipe=1, data=92, model=1): 441, ProcessCoord(pipe=1, data=93, model=0): 442, ProcessCoord(pipe=1, data=93, model=1): 443, ProcessCoord(pipe=1, data=94, model=0): 444, ProcessCoord(pipe=1, data=94, model=1): 445, ProcessCoord(pipe=1, data=95, model=0): 446, ProcessCoord(pipe=1, data=95, model=1): 447, ProcessCoord(pipe=1, data=96, model=0): 448, ProcessCoord(pipe=1, data=96, model=1): 449, ProcessCoord(pipe=1, data=97, model=0): 450, ProcessCoord(pipe=1, data=97, model=1): 451, ProcessCoord(pipe=1, data=98, model=0): 452, ProcessCoord(pipe=1, data=98, model=1): 453, ProcessCoord(pipe=1, data=99, model=0): 454, ProcessCoord(pipe=1, data=99, model=1): 455, ProcessCoord(pipe=1, data=100, model= + 0: 0): 456, ProcessCoord(pipe=1, data=100, model=1): 457, ProcessCoord(pipe=1, data=101, model=0): 458, ProcessCoord(pipe=1, data=101, model=1): 459, ProcessCoord(pipe=1, data=102, model=0): 460, ProcessCoord(pipe=1, data=102, model=1): 461, ProcessCoord(pipe=1, data=103, model=0): 462, ProcessCoord(pipe=1, data=103, model=1): 463, ProcessCoord(pipe=1, data=104, model=0): 464, ProcessCoord(pipe=1, data=104, model=1): 465, ProcessCoord(pipe=1, data=105, model=0): 466, ProcessCoord(pipe=1, data=105, model=1): 467, ProcessCoord(pipe=1, data=106, model=0): 468, ProcessCoord(pipe=1, data=106, model=1): 469, ProcessCoord(pipe=1, data=107, model=0): 470, ProcessCoord(pipe=1, data=107, model=1): 471, ProcessCoord(pipe=1, data=108, model=0): 472, ProcessCoord(pipe=1, data=108, model=1): 473, ProcessCoord(pipe=1, data=109, model=0): 474, ProcessCoord(pipe=1, data=109, model=1): 475, ProcessCoord(pipe=1, data=110, model=0): 476, ProcessCoord(pipe=1, data=110, model=1): 477, ProcessCoord(pipe=1, data=111, model=0): 478, Pro + 0: cessCoord(pipe=1, data=111, model=1): 479, ProcessCoord(pipe=1, data=112, model=0): 480, ProcessCoord(pipe=1, data=112, model=1): 481, ProcessCoord(pipe=1, data=113, model=0): 482, ProcessCoord(pipe=1, data=113, model=1): 483, ProcessCoord(pipe=1, data=114, model=0): 484, ProcessCoord(pipe=1, data=114, model=1): 485, ProcessCoord(pipe=1, data=115, model=0): 486, ProcessCoord(pipe=1, data=115, model=1): 487, ProcessCoord(pipe=1, data=116, model=0): 488, ProcessCoord(pipe=1, data=116, model=1): 489, ProcessCoord(pipe=1, data=117, model=0): 490, ProcessCoord(pipe=1, data=117, model=1): 491, ProcessCoord(pipe=1, data=118, model=0): 492, ProcessCoord(pipe=1, data=118, model=1): 493, ProcessCoord(pipe=1, data=119, model=0): 494, ProcessCoord(pipe=1, data=119, model=1): 495, ProcessCoord(pipe=1, data=120, model=0): 496, ProcessCoord(pipe=1, data=120, model=1): 497, ProcessCoord(pipe=1, data=121, model=0): 498, ProcessCoord(pipe=1, data=121, model=1): 499, ProcessCoord(pipe=1, data=122, model=0): 500, ProcessCoord(pi + 0: pe=1, data=122, model=1): 501, ProcessCoord(pipe=1, data=123, model=0): 502, ProcessCoord(pipe=1, data=123, model=1): 503, ProcessCoord(pipe=1, data=124, model=0): 504, ProcessCoord(pipe=1, data=124, model=1): 505, ProcessCoord(pipe=1, data=125, model=0): 506, ProcessCoord(pipe=1, data=125, model=1): 507, ProcessCoord(pipe=1, data=126, model=0): 508, ProcessCoord(pipe=1, data=126, model=1): 509, ProcessCoord(pipe=1, data=127, model=0): 510, ProcessCoord(pipe=1, data=127, model=1): 511} + 0: [2023-04-29 09:44:52,833] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=24 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: stage=1 layers=25 + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: ParallelTransformerLayerPipe + 0: 38: ParallelTransformerLayerPipe + 0: 39: ParallelTransformerLayerPipe + 0: 40: ParallelTransformerLayerPipe + 0: 41: ParallelTransformerLayerPipe + 0: 42: ParallelTransformerLayerPipe + 0: 43: ParallelTransformerLayerPipe + 0: 44: ParallelTransformerLayerPipe + 0: 45: undo + 0: 46: MixedFusedLayerNorm + 0: 47: EmbeddingPipe + 0: 48: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-04-29 09:44:56,588] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-04-29 09:44:56,589] [INFO] [utils.py:828:see_memory_usage] MA 4.16 GB Max_MA 4.16 GB CA 4.17 GB Max_CA 4 GB + 0: [2023-04-29 09:44:56,589] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.4 GB, percent = 7.8% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-04-29 09:44:56,591] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +32: ninja: no work to do. +48: Time to load utils op: 0.10853815078735352 seconds +50: Time to load utils op: 0.10756444931030273 secondsTime to load utils op: 0.10755705833435059 secondsTime to load utils op: 0.10756754875183105 seconds +50: +50: +50: Time to load utils op: 0.10766339302062988 seconds +49: Time to load utils op: 0.10886812210083008 secondsTime to load utils op: 0.1086266040802002 seconds +49: +49: Time to load utils op: 0.10780000686645508 seconds +49: Time to load utils op: 0.10846376419067383 seconds +32: Time to load utils op: 0.20451712608337402 secondsTime to load utils op: 0.20406556129455566 secondsTime to load utils op: 0.20444059371948242 seconds +32: +32: +32: Time to load utils op: 0.22267699241638184 seconds +49: Time to load utils op: 0.20630574226379395 seconds +49: Time to load utils op: 0.20602202415466309 seconds +49: Time to load utils op: 0.20647931098937988 seconds +48: Time to load utils op: 0.2061920166015625 seconds +48: Time to load utils op: 0.20625042915344238 seconds +48: Time to load utils op: 0.20632338523864746 seconds +49: Time to load utils op: 0.20675134658813477 seconds +48: Time to load utils op: 0.2064955234527588 seconds +54: Time to load utils op: 0.1198418140411377 seconds +54: Time to load utils op: 0.205643892288208 secondsTime to load utils op: 0.11940193176269531 seconds +54: +54: Time to load utils op: 0.11919164657592773 secondsTime to load utils op: 0.12000346183776855 seconds +54: +54: Time to load utils op: 0.20517396926879883 seconds +54: Time to load utils op: 0.20547842979431152 seconds +54: Time to load utils op: 0.20536184310913086 seconds +50: Time to load utils op: 0.2050001621246338 seconds +50: Time to load utils op: 0.20519804954528809 seconds +50: Time to load utils op: 0.20534610748291016 secondsTime to load utils op: 0.2053508758544922 seconds +50: +52: Time to load utils op: 0.20899748802185059 seconds +52: Time to load utils op: 0.20901083946228027 seconds +52: Time to load utils op: 0.1263139247894287 seconds +52: Time to load utils op: 0.12630629539489746 secondsTime to load utils op: 0.12628698348999023 seconds +52: +52: Time to load utils op: 0.20902776718139648 seconds +52: Time to load utils op: 0.1277613639831543 seconds +52: Time to load utils op: 0.20903658866882324 seconds +51: Time to load utils op: 0.21076750755310059 secondsTime to load utils op: 0.1301593780517578 seconds +51: +51: Time to load utils op: 0.21077919006347656 seconds +51: Time to load utils op: 0.12993311882019043 secondsTime to load utils op: 0.12944817543029785 seconds +51: Time to load utils op: 0.12996959686279297 secondsTime to load utils op: 0.21078276634216309 seconds +51: Time to load utils op: 0.2107834815979004 seconds +51: +51: +53: Time to load utils op: 0.12988615036010742 secondsTime to load utils op: 0.21231532096862793 secondsTime to load utils op: 0.12988543510437012 seconds +53: +53: +53: Time to load utils op: 0.21231365203857422 secondsTime to load utils op: 0.2123098373413086 secondsTime to load utils op: 0.12986350059509277 seconds +53: +53: +53: Time to load utils op: 0.12989211082458496 seconds +53: Time to load utils op: 0.21233010292053223 seconds +58: Time to load utils op: 0.21148324012756348 secondsTime to load utils op: 0.21148228645324707 secondsTime to load utils op: 0.13055753707885742 secondsTime to load utils op: 0.13010549545288086 seconds +58: +58: +58: Time to load utils op: 0.13000035285949707 seconds +58: Time to load utils op: 0.2115006446838379 seconds +58: +58: Time to load utils op: 0.13018250465393066 secondsTime to load utils op: 0.21149182319641113 seconds +58: +40: Time to load utils op: 0.22330665588378906 secondsTime to load utils op: 0.22331643104553223 secondsTime to load utils op: 0.2233114242553711 seconds +40: +40: +40: Time to load utils op: 0.2233285903930664 seconds +55: Time to load utils op: 0.21683597564697266 seconds +55: Time to load utils op: 0.21676993370056152 secondsTime to load utils op: 0.1341381072998047 secondsTime to load utils op: 0.13413691520690918 seconds +55: +55: +55: Time to load utils op: 0.21684741973876953 secondsTime to load utils op: 0.13412165641784668 seconds +55: +55: Time to load utils op: 0.21685218811035156 secondsTime to load utils op: 0.1341567039489746 seconds +55: +63: Time to load utils op: 0.21748113632202148 secondsTime to load utils op: 0.13219976425170898 seconds +63: +63: Time to load utils op: 0.1320340633392334 seconds +63: Time to load utils op: 0.21760129928588867 seconds +63: Time to load utils op: 0.21749401092529297 secondsTime to load utils op: 0.13152217864990234 secondsTime to load utils op: 0.21767878532409668 seconds +63: +63: Time to load utils op: 0.13155245780944824 seconds +63: +56: Time to load utils op: 0.13474559783935547 secondsTime to load utils op: 0.21675872802734375 secondsTime to load utils op: 0.13474321365356445 secondsTime to load utils op: 0.21675944328308105 secondsTime to load utils op: 0.1347196102142334 seconds +56: +56: +56: +56: +56: Time to load utils op: 0.1347362995147705 seconds +56: Time to load utils op: 0.21677207946777344 seconds +56: Time to load utils op: 0.21676898002624512 seconds +61: Time to load utils op: 0.21827363967895508 secondsTime to load utils op: 0.13370108604431152 secondsTime to load utils op: 0.13398361206054688 seconds +61: +61: +61: Time to load utils op: 0.21900677680969238 secondsTime to load utils op: 0.21961236000061035 seconds +61: +61: Time to load utils op: 0.21962189674377441 secondsTime to load utils op: 0.13415908813476562 seconds +61: +61: Time to load utils op: 0.13374829292297363 seconds +33: Time to load utils op: 0.23400473594665527 seconds +33: Time to load utils op: 0.23404860496520996 secondsTime to load utils op: 0.23402810096740723 seconds +33: Time to load utils op: 0.23461365699768066 seconds +33: +47: Time to load utils op: 0.22827386856079102 secondsTime to load utils op: 0.22735857963562012 secondsTime to load utils op: 0.22818493843078613 seconds +47: Time to load utils op: 0.22722983360290527 seconds +47: +47: +34: Time to load utils op: 0.2344520092010498 secondsTime to load utils op: 0.23441648483276367 seconds +34: Time to load utils op: 0.2344224452972412 seconds +34: +34: Time to load utils op: 0.2346811294555664 seconds +57: Time to load utils op: 0.21928691864013672 secondsTime to load utils op: 0.21929550170898438 secondsTime to load utils op: 0.21929645538330078 seconds +57: Time to load utils op: 0.13825082778930664 seconds +57: +57: +57: Time to load utils op: 0.21930289268493652 secondsTime to load utils op: 0.13820910453796387 seconds +57: +57: Time to load utils op: 0.13821625709533691 secondsTime to load utils op: 0.13823604583740234 seconds +57: +60: Time to load utils op: 0.1383368968963623 seconds +60: Time to load utils op: 0.2232377529144287 secondsTime to load utils op: 0.13845324516296387 secondsTime to load utils op: 0.13831210136413574 seconds +60: +60: +60: Time to load utils op: 0.13808155059814453 secondsTime to load utils op: 0.22324657440185547 seconds +60: +60: Time to load utils op: 0.22309279441833496 seconds +60: Time to load utils op: 0.2227785587310791 seconds +59: Time to load utils op: 0.22154569625854492 seconds +59: Time to load utils op: 0.22155141830444336 seconds +59: Time to load utils op: 0.13894915580749512 seconds +59: Time to load utils op: 0.22157025337219238 secondsTime to load utils op: 0.2215712070465088 seconds +59: Time to load utils op: 0.1389484405517578 secondsTime to load utils op: 0.13891196250915527 seconds +59: +59: +59: Time to load utils op: 0.13933348655700684 seconds +36: Time to load utils op: 0.2340559959411621 secondsTime to load utils op: 0.23405933380126953 secondsTime to load utils op: 0.2321147918701172 secondsTime to load utils op: 0.23404979705810547 seconds +36: +36: +36: +62: Time to load utils op: 0.22029399871826172 secondsTime to load utils op: 0.13852739334106445 secondsTime to load utils op: 0.13862133026123047 seconds +62: +62: +62: Time to load utils op: 0.22030878067016602 seconds +45: Time to load utils op: 0.22979307174682617 secondsTime to load utils op: 0.22979187965393066 secondsTime to load utils op: 0.2297959327697754 secondsTime to load utils op: 0.2297978401184082 seconds +45: +45: +45: +62: Time to load utils op: 0.22031307220458984 seconds +62: Time to load utils op: 0.2203223705291748 secondsTime to load utils op: 0.13855242729187012 seconds +62: Time to load utils op: 0.13852238655090332 seconds +62: +35: Time to load utils op: 0.23550963401794434 secondsTime to load utils op: 0.23550701141357422 secondsTime to load utils op: 0.23551154136657715 seconds +35: +35: +35: Time to load utils op: 0.2355201244354248 seconds +37: Time to load utils op: 0.23439884185791016 secondsTime to load utils op: 0.23440265655517578 seconds +37: +37: Time to load utils op: 0.23441171646118164 seconds +37: Time to load utils op: 0.2344045639038086 seconds +43: Time to load utils op: 0.23640990257263184 secondsTime to load utils op: 0.23630475997924805 secondsTime to load utils op: 0.23620820045471191 secondsTime to load utils op: 0.23691248893737793 seconds +43: +43: +43: +39: Time to load utils op: 0.23482036590576172 secondsTime to load utils op: 0.2348337173461914 seconds +39: +39: Time to load utils op: 0.2348334789276123 seconds +39: Time to load utils op: 0.234849214553833 seconds +42: Time to load utils op: 0.25231266021728516 seconds +42: Time to load utils op: 0.25232410430908203 seconds +42: Time to load utils op: 0.25234293937683105 seconds +42: Time to load utils op: 0.25235986709594727 seconds +41: Time to load utils op: 0.2565169334411621 secondsTime to load utils op: 0.25652265548706055 secondsTime to load utils op: 0.2565188407897949 seconds +41: +41: +41: Time to load utils op: 0.25646233558654785 seconds +46: Time to load utils op: 0.2544887065887451 seconds +46: Time to load utils op: 0.254504919052124 seconds +46: Time to load utils op: 0.25452232360839844 seconds +46: Time to load utils op: 0.25454139709472656 seconds +38: Time to load utils op: 0.25940871238708496 seconds +38: Time to load utils op: 0.25943422317504883 seconds +44: Time to load utils op: 0.2560126781463623 seconds +38: Time to load utils op: 0.2594482898712158 seconds +38: Time to load utils op: 0.2594611644744873 seconds +44: Time to load utils op: 0.25601720809936523 secondsTime to load utils op: 0.25603342056274414 seconds +44: +44: Time to load utils op: 0.25604248046875 seconds +32: Time to load utils op: 0.20224738121032715 seconds +32: Time to load utils op: 0.20200300216674805 seconds +32: Time to load utils op: 0.20202159881591797 seconds +32: Time to load utils op: 0.20205020904541016 seconds +34: Time to load utils op: 0.2022554874420166 seconds +33: Time to load utils op: 0.20290327072143555 seconds +34: Time to load utils op: 0.20181822776794434 seconds +33: Time to load utils op: 0.202301025390625 seconds +33: Time to load utils op: 0.2022690773010254 seconds +34: Time to load utils op: 0.20179533958435059 seconds +34: Time to load utils op: 0.20209956169128418 seconds +33: Time to load utils op: 0.2026047706604004 seconds +36: Time to load utils op: 0.20209670066833496 seconds +36: Time to load utils op: 0.20200896263122559 seconds +35: Time to load utils op: 0.202467679977417 seconds +36: Time to load utils op: 0.20201396942138672 seconds +35: Time to load utils op: 0.20272159576416016 seconds +36: Time to load utils op: 0.20206332206726074 seconds +35: Time to load utils op: 0.2028353214263916 seconds +35: Time to load utils op: 0.20287227630615234 seconds +37: Time to load utils op: 0.20270490646362305 seconds +37: Time to load utils op: 0.20276546478271484 seconds +37: Time to load utils op: 0.20290493965148926 seconds +37: Time to load utils op: 0.2029709815979004 seconds +38: Time to load utils op: 0.20299029350280762 seconds +38: Time to load utils op: 0.2030947208404541 seconds +38: Time to load utils op: 0.2032475471496582 seconds +39: Time to load utils op: 0.20306777954101562 seconds +38: Time to load utils op: 0.20341014862060547 seconds +39: Time to load utils op: 0.2028195858001709 seconds +39: Time to load utils op: 0.2030181884765625 seconds +39: Time to load utils op: 0.20307302474975586 seconds +40: Time to load utils op: 0.20242857933044434 seconds +40: Time to load utils op: 0.20223355293273926 seconds +40: Time to load utils op: 0.2023005485534668 seconds +40: Time to load utils op: 0.2023601531982422 seconds +41: Time to load utils op: 0.20265746116638184 seconds +43: Time to load utils op: 0.20222783088684082 seconds +41: Time to load utils op: 0.20282578468322754 seconds +41: Time to load utils op: 0.20298337936401367 seconds +41: Time to load utils op: 0.20290374755859375 seconds +42: Time to load utils op: 0.20234036445617676 seconds +43: Time to load utils op: 0.20230722427368164 seconds +43: Time to load utils op: 0.20206499099731445 seconds +42: Time to load utils op: 0.20259809494018555 seconds +42: Time to load utils op: 0.20266294479370117 seconds +42: Time to load utils op: 0.2026517391204834 seconds +43: Time to load utils op: 0.20224666595458984 seconds +44: Time to load utils op: 0.2022237777709961 seconds +44: Time to load utils op: 0.2023329734802246 seconds +44: Time to load utils op: 0.20273685455322266 seconds +44: Time to load utils op: 0.20280671119689941 seconds +45: Time to load utils op: 0.20294904708862305 seconds +45: Time to load utils op: 0.20282196998596191 seconds +45: Time to load utils op: 0.20284199714660645 seconds +47: Time to load utils op: 0.20296788215637207 seconds +45: Time to load utils op: 0.2029879093170166 seconds +46: Time to load utils op: 0.20238184928894043 seconds +48: Time to load utils op: 0.20219731330871582 seconds +46: Time to load utils op: 0.20260095596313477 seconds +46: Time to load utils op: 0.20274949073791504 seconds +46: Time to load utils op: 0.20301032066345215 seconds +47: Time to load utils op: 0.20281696319580078 seconds +48: Time to load utils op: 0.20224952697753906 secondsTime to load utils op: 0.20225048065185547 seconds +48: +47: Time to load utils op: 0.2029130458831787 seconds +47: Time to load utils op: 0.2028963565826416 seconds +32: Time to load utils op: 0.0005214214324951172 seconds +32: Time to load utils op: 0.0005507469177246094 secondsTime to load utils op: 0.0004553794860839844 secondsTime to load utils op: 0.0004773139953613281 secondsTime to load utils op: 0.0005116462707519531 secondsTime to load utils op: 0.0004343986511230469 seconds +32: +32: +32: +32: +32: Time to load utils op: 0.0004913806915283203 seconds +32: Time to load utils op: 0.0005247592926025391 seconds +50: Time to load utils op: 0.0005233287811279297 seconds +50: Time to load utils op: 0.0005042552947998047 seconds +50: Time to load utils op: 0.0005464553833007812 seconds +50: Time to load utils op: 0.0004875659942626953 secondsTime to load utils op: 0.0004737377166748047 secondsTime to load utils op: 0.0004818439483642578 seconds +50: +50: +50: Time to load utils op: 0.0005040168762207031 seconds +50: Time to load utils op: 0.00046062469482421875 seconds +41: Time to load utils op: 0.00042176246643066406 seconds +41: Time to load utils op: 0.000507354736328125 seconds +49: Time to load utils op: 0.0005092620849609375 seconds +49: Time to load utils op: 0.0004839897155761719 seconds +49: Time to load utils op: 0.00039124488830566406 secondsTime to load utils op: 0.00042057037353515625 seconds +49: +49: Time to load utils op: 0.0003604888916015625 seconds +49: Time to load utils op: 0.0005321502685546875 seconds +49: Time to load utils op: 0.0005457401275634766 secondsTime to load utils op: 0.0004031658172607422 seconds +49: +43: Time to load utils op: 0.0005202293395996094 seconds +43: Time to load utils op: 0.00045490264892578125 seconds +43: Time to load utils op: 0.0004277229309082031 seconds +43: Time to load utils op: 0.0004391670227050781 seconds +43: Time to load utils op: 0.0005433559417724609 seconds +43: Time to load utils op: 0.0005843639373779297 seconds +43: Time to load utils op: 0.0005898475646972656 seconds +43: Time to load utils op: 0.0005536079406738281 seconds +47: Time to load utils op: 0.0005178451538085938 seconds +47: Time to load utils op: 0.0004134178161621094 seconds +47: Time to load utils op: 0.00043511390686035156 seconds +47: Time to load utils op: 0.0005571842193603516 seconds +47: Time to load utils op: 0.0004661083221435547 seconds +47: Time to load utils op: 0.0004558563232421875 seconds +41: Time to load utils op: 0.0006074905395507812 seconds +41: Time to load utils op: 0.00044345855712890625 seconds +41: Time to load utils op: 0.0004425048828125 seconds +41: Time to load utils op: 0.00047087669372558594 seconds +41: Time to load utils op: 0.0004794597625732422 seconds +41: Time to load utils op: 0.0004918575286865234 seconds +45: Time to load utils op: 0.0005466938018798828 seconds +45: Time to load utils op: 0.0004096031188964844 seconds +45: Time to load utils op: 0.0006597042083740234 seconds +45: Time to load utils op: 0.000640869140625 secondsTime to load utils op: 0.0006639957427978516 secondsTime to load utils op: 0.0006372928619384766 seconds +45: Time to load utils op: 0.0006532669067382812 seconds +45: +45: +45: Time to load utils op: 0.0005025863647460938 seconds +47: Time to load utils op: 0.00041866302490234375 secondsTime to load utils op: 0.0003898143768310547 seconds +47: +61: Time to load utils op: 0.0005373954772949219 seconds +61: Time to load utils op: 0.0004456043243408203 seconds +61: Time to load utils op: 0.0005371570587158203 seconds +61: Time to load utils op: 0.0004894733428955078 seconds +61: Time to load utils op: 0.00044608116149902344 secondsTime to load utils op: 0.0005116462707519531 seconds +61: Time to load utils op: 0.0004792213439941406 seconds +61: +61: Time to load utils op: 0.00047326087951660156 seconds +62: Time to load utils op: 0.0005276203155517578 seconds +62: Time to load utils op: 0.0004897117614746094 seconds +62: Time to load utils op: 0.0005567073822021484 seconds +62: Time to load utils op: 0.0005548000335693359 secondsTime to load utils op: 0.0005686283111572266 secondsTime to load utils op: 0.0005681514739990234 seconds +62: +62: +62: Time to load utils op: 0.0006878376007080078 secondsTime to load utils op: 0.0006225109100341797 seconds +62: +56: Time to load utils op: 0.0005009174346923828 seconds +56: Time to load utils op: 0.00040650367736816406 seconds +56: Time to load utils op: 0.0004076957702636719 seconds +56: Time to load utils op: 0.0004100799560546875 seconds +56: Time to load utils op: 0.0004124641418457031 seconds +56: Time to load utils op: 0.0004324913024902344 secondsTime to load utils op: 0.0005309581756591797 seconds +56: +56: Time to load utils op: 0.00042176246643066406 seconds +40: Time to load utils op: 0.0005679130554199219 seconds +40: Time to load utils op: 0.0004334449768066406 seconds +40: Time to load utils op: 0.0005846023559570312 seconds +40: Time to load utils op: 0.0005664825439453125 seconds +40: Time to load utils op: 0.000598907470703125 secondsTime to load utils op: 0.0005767345428466797 seconds +40: +40: Time to load utils op: 0.0007579326629638672 seconds +40: Time to load utils op: 0.0007436275482177734 seconds +55: Time to load utils op: 0.0005447864532470703 secondsTime to load utils op: 0.0005753040313720703 seconds +55: +55: Time to load utils op: 0.0005648136138916016 seconds +55: Time to load utils op: 0.0005810260772705078 seconds +55: Time to load utils op: 0.0006091594696044922 seconds +55: Time to load utils op: 0.0005624294281005859 seconds +55: Time to load utils op: 0.0006005764007568359 seconds +55: Time to load utils op: 0.0006968975067138672 seconds +36: Time to load utils op: 0.0005578994750976562 seconds +36: Time to load utils op: 0.0005941390991210938 seconds +36: Time to load utils op: 0.0006108283996582031 seconds +36: Time to load utils op: 0.0006101131439208984 secondsTime to load utils op: 0.0006041526794433594 seconds +36: +36: Time to load utils op: 0.0005812644958496094 seconds +36: Time to load utils op: 0.0006031990051269531 seconds +36: Time to load utils op: 0.0006108283996582031 seconds +48: Time to load utils op: 0.0005154609680175781 secondsTime to load utils op: 0.000522613525390625 seconds +48: +48: Time to load utils op: 0.0005290508270263672 seconds +48: Time to load utils op: 0.000461578369140625 secondsTime to load utils op: 0.0005726814270019531 secondsTime to load utils op: 0.0005896091461181641 secondsTime to load utils op: 0.0005650520324707031 seconds +48: +48: +48: +33: Time to load utils op: 0.0005314350128173828 seconds +48: Time to load utils op: 0.0006184577941894531 seconds +33: Time to load utils op: 0.0003993511199951172 seconds +33: Time to load utils op: 0.0005638599395751953 seconds +33: Time to load utils op: 0.0004482269287109375 seconds +33: Time to load utils op: 0.00054931640625 seconds +33: Time to load utils op: 0.0005207061767578125 seconds +33: Time to load utils op: 0.0005362033843994141 seconds +33: Time to load utils op: 0.0006902217864990234 seconds +46: Time to load utils op: 0.0005345344543457031 seconds +46: Time to load utils op: 0.00040459632873535156 seconds +46: Time to load utils op: 0.0005280971527099609 seconds +46: Time to load utils op: 0.00039124488830566406 seconds +59: Time to load utils op: 0.0005121231079101562 seconds +59: Time to load utils op: 0.0005276203155517578 seconds +59: Time to load utils op: 0.0005612373352050781 seconds +46: Time to load utils op: 0.00044417381286621094 seconds +46: Time to load utils op: 0.0004143714904785156 seconds +59: Time to load utils op: 0.0005650520324707031 seconds +59: Time to load utils op: 0.0005736351013183594 secondsTime to load utils op: 0.0006017684936523438 seconds +59: +59: Time to load utils op: 0.0005838871002197266 seconds +46: Time to load utils op: 0.00041985511779785156 seconds +46: Time to load utils op: 0.0004317760467529297 seconds +59: Time to load utils op: 0.0006468296051025391 seconds +52: Time to load utils op: 0.0005071163177490234 secondsTime to load utils op: 0.0005092620849609375 seconds +52: +58: Time to load utils op: 0.0005183219909667969 secondsTime to load utils op: 0.0004086494445800781 seconds +58: +52: Time to load utils op: 0.00045299530029296875 seconds +58: Time to load utils op: 0.0005135536193847656 seconds +58: Time to load utils op: 0.0005714893341064453 seconds +52: Time to load utils op: 0.000431060791015625 seconds +58: Time to load utils op: 0.0005762577056884766 seconds +58: Time to load utils op: 0.0005731582641601562 secondsTime to load utils op: 0.0005564689636230469 secondsTime to load utils op: 0.0005662441253662109 seconds +52: Time to load utils op: 0.00042557716369628906 seconds +58: +58: +52: Time to load utils op: 0.0005028247833251953 seconds +52: Time to load utils op: 0.0004353523254394531 secondsTime to load utils op: 0.0004296302795410156 seconds +52: +35: Time to load utils op: 0.0005352497100830078 seconds +35: Time to load utils op: 0.0005698204040527344 secondsTime to load utils op: 0.0006029605865478516 seconds +35: Time to load utils op: 0.0005908012390136719 seconds +35: +35: Time to load utils op: 0.00047397613525390625 seconds +51: Time to load utils op: 0.0003974437713623047 seconds +35: Time to load utils op: 0.0005578994750976562 seconds +35: Time to load utils op: 0.0006105899810791016 seconds +51: Time to load utils op: 0.0005507469177246094 secondsTime to load utils op: 0.0005626678466796875 seconds +51: +35: Time to load utils op: 0.0005979537963867188 seconds +51: Time to load utils op: 0.0003991127014160156 seconds +51: Time to load utils op: 0.0003981590270996094 seconds +51: Time to load utils op: 0.0005459785461425781 seconds +51: Time to load utils op: 0.0005819797515869141 secondsTime to load utils op: 0.0005536079406738281 seconds +51: +53: Time to load utils op: 0.0005085468292236328 secondsTime to load utils op: 0.0004169940948486328 seconds +53: +53: Time to load utils op: 0.00041413307189941406 seconds +53: Time to load utils op: 0.00042319297790527344 seconds +53: Time to load utils op: 0.0004754066467285156 seconds +53: Time to load utils op: 0.00041675567626953125 seconds +53: Time to load utils op: 0.00042057037353515625 seconds +53: Time to load utils op: 0.00038313865661621094 seconds +63: Time to load utils op: 0.00047135353088378906 seconds +63: Time to load utils op: 0.0004909038543701172 seconds +63: Time to load utils op: 0.0004489421844482422 seconds +63: Time to load utils op: 0.0005407333374023438 secondsTime to load utils op: 0.0005679130554199219 seconds +63: +63: Time to load utils op: 0.0005884170532226562 secondsTime to load utils op: 0.0005776882171630859 seconds +63: Time to load utils op: 0.0005688667297363281 seconds +63: +54: Time to load utils op: 0.0005335807800292969 seconds +54: Time to load utils op: 0.00043845176696777344 secondsTime to load utils op: 0.0004360675811767578 seconds +54: +54: Time to load utils op: 0.00044345855712890625 secondsTime to load utils op: 0.0004165172576904297 seconds +54: +54: Time to load utils op: 0.0005164146423339844 seconds +54: Time to load utils op: 0.00041794776916503906 seconds +54: Time to load utils op: 0.00041484832763671875 seconds +60: Time to load utils op: 0.0005373954772949219 seconds +60: Time to load utils op: 0.0006036758422851562 seconds +60: Time to load utils op: 0.0006208419799804688 seconds +60: Time to load utils op: 0.0005786418914794922 secondsTime to load utils op: 0.0006263256072998047 seconds +60: Time to load utils op: 0.0006048679351806641 secondsTime to load utils op: 0.0006120204925537109 seconds +60: +60: +60: Time to load utils op: 0.0006961822509765625 seconds +42: Time to load utils op: 0.0005719661712646484 seconds +42: Time to load utils op: 0.0005762577056884766 seconds +42: Time to load utils op: 0.0005881786346435547 seconds +42: Time to load utils op: 0.0005819797515869141 seconds +42: Time to load utils op: 0.0005743503570556641 seconds +42: Time to load utils op: 0.0005893707275390625 seconds +42: Time to load utils op: 0.0005981922149658203 secondsTime to load utils op: 0.0005710124969482422 seconds +42: +57: Time to load utils op: 0.0005426406860351562 seconds +57: Time to load utils op: 0.0005218982696533203 secondsTime to load utils op: 0.0005311965942382812 seconds +57: +57: Time to load utils op: 0.0005636215209960938 seconds +57: Time to load utils op: 0.0006182193756103516 seconds +57: Time to load utils op: 0.0005986690521240234 secondsTime to load utils op: 0.0006132125854492188 seconds +57: +57: Time to load utils op: 0.0005931854248046875 seconds +34: Time to load utils op: 0.0004947185516357422 seconds +34: Time to load utils op: 0.0004470348358154297 secondsTime to load utils op: 0.00040841102600097656 secondsTime to load utils op: 0.00040149688720703125 seconds +34: +34: +38: Time to load utils op: 0.0004417896270751953 seconds +38: Time to load utils op: 0.00044465065002441406 seconds +38: Time to load utils op: 0.000507354736328125 seconds +34: Time to load utils op: 0.0005047321319580078 seconds +34: Time to load utils op: 0.0004215240478515625 seconds +38: Time to load utils op: 0.00044989585876464844 secondsTime to load utils op: 0.00046062469482421875 seconds +38: +34: Time to load utils op: 0.0005309581756591797 secondsTime to load utils op: 0.0005464553833007812 seconds +34: +37: Time to load utils op: 0.0005288124084472656 seconds +38: Time to load utils op: 0.0005548000335693359 seconds +37: Time to load utils op: 0.00040531158447265625 seconds +37: Time to load utils op: 0.0005388259887695312 secondsTime to load utils op: 0.0004684925079345703 seconds +37: +38: Time to load utils op: 0.0005583763122558594 seconds +37: Time to load utils op: 0.0004627704620361328 seconds +38: Time to load utils op: 0.0005750656127929688 seconds +37: Time to load utils op: 0.0005993843078613281 seconds +37: Time to load utils op: 0.0004527568817138672 seconds +37: Time to load utils op: 0.0005831718444824219 seconds +39: Time to load utils op: 0.0004267692565917969 secondsTime to load utils op: 0.0005488395690917969 seconds +39: +39: Time to load utils op: 0.0004889965057373047 seconds +39: Time to load utils op: 0.0004489421844482422 seconds +39: Time to load utils op: 0.0004296302795410156 secondsTime to load utils op: 0.00042939186096191406 seconds +39: +39: Time to load utils op: 0.0004265308380126953 seconds +39: Time to load utils op: 0.0004742145538330078 seconds +44: Time to load utils op: 0.0005550384521484375 secondsTime to load utils op: 0.0005548000335693359 secondsTime to load utils op: 0.0003933906555175781 seconds +44: +44: +44: Time to load utils op: 0.0004088878631591797 seconds +44: Time to load utils op: 0.0005688667297363281 seconds +44: Time to load utils op: 0.0005259513854980469 seconds +44: Time to load utils op: 0.0005624294281005859 seconds +44: Time to load utils op: 0.0006754398345947266 seconds + 0: [2023-04-29 09:45:01,456] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-04-29 09:45:01,457] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-04-29 09:45:01,457] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-04-29 09:45:01,465] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-04-29 09:45:01,465] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-04-29 09:45:01,590] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-04-29 09:45:01,591] [INFO] [utils.py:828:see_memory_usage] MA 4.15 GB Max_MA 4.18 GB CA 4.18 GB Max_CA 4 GB + 0: [2023-04-29 09:45:01,591] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.0 GB, percent = 7.9% + 0: ninja: no work to do. + 0: Time to load utils op: 0.18141913414001465 seconds + 0: Time to load utils op: 0.10254430770874023 seconds + 0: Time to load utils op: 0.2046034336090088 seconds + 0: Time to load utils op: 0.2052474021911621 seconds + 1: Time to load utils op: 0.2126140594482422 seconds + 1: Time to load utils op: 0.2125406265258789 secondsTime to load utils op: 0.2122209072113037 seconds + 1: + 1: Time to load utils op: 0.21196293830871582 seconds + 2: Time to load utils op: 0.21097970008850098 secondsTime to load utils op: 0.21085262298583984 secondsTime to load utils op: 0.21110320091247559 secondsTime to load utils op: 0.21092486381530762 seconds + 2: + 2: + 2: + 5: Time to load utils op: 0.21053743362426758 secondsTime to load utils op: 0.21030974388122559 secondsTime to load utils op: 0.21070408821105957 seconds + 5: + 5: Time to load utils op: 0.21019744873046875 seconds + 5: + 8: Time to load utils op: 0.20991921424865723 secondsTime to load utils op: 0.21008872985839844 secondsTime to load utils op: 0.20995879173278809 secondsTime to load utils op: 0.21006369590759277 seconds + 8: + 8: + 8: + 3: Time to load utils op: 0.2289717197418213 secondsTime to load utils op: 0.22921538352966309 seconds + 3: Time to load utils op: 0.22910380363464355 seconds + 3: + 3: Time to load utils op: 0.22902727127075195 seconds + 6: Time to load utils op: 0.22582435607910156 secondsTime to load utils op: 0.22582578659057617 secondsTime to load utils op: 0.22582340240478516 seconds + 6: + 6: + 6: Time to load utils op: 0.2258305549621582 seconds + 7: Time to load utils op: 0.22804045677185059 secondsTime to load utils op: 0.22804474830627441 secondsTime to load utils op: 0.2280442714691162 seconds + 7: + 7: + 7: Time to load utils op: 0.22803854942321777 seconds +14: Time to load utils op: 0.22857999801635742 secondsTime to load utils op: 0.22779560089111328 secondsTime to load utils op: 0.22841954231262207 seconds +14: +14: +14: Time to load utils op: 0.2283189296722412 seconds + 0: Time to load utils op: 0.0004973411560058594 seconds + 0: Time to load utils op: 0.0005612373352050781 seconds + 0: Time to load utils op: 0.0005025863647460938 seconds + 4: Time to load utils op: 0.23067450523376465 secondsTime to load utils op: 0.23067355155944824 seconds + 4: + 4: Time to load utils op: 0.23067331314086914 seconds + 4: Time to load utils op: 0.23070907592773438 seconds +13: Time to load utils op: 0.23108172416687012 secondsTime to load utils op: 0.2310800552368164 secondsTime to load utils op: 0.2307734489440918 secondsTime to load utils op: 0.23113703727722168 seconds +13: +13: +13: + 1: Time to load utils op: 0.0010461807250976562 seconds + 1: Time to load utils op: 0.0010449886322021484 seconds + 1: Time to load utils op: 0.0010628700256347656 seconds + 1: Time to load utils op: 0.0010890960693359375 seconds + 5: Time to load utils op: 0.0008051395416259766 seconds + 5: Time to load utils op: 0.0008671283721923828 seconds + 5: Time to load utils op: 0.0009124279022216797 seconds + 5: Time to load utils op: 0.0009353160858154297 seconds + 2: Time to load utils op: 0.0006117820739746094 seconds + 8: Time to load utils op: 0.0005714893341064453 seconds + 8: Time to load utils op: 0.0005452632904052734 secondsTime to load utils op: 0.0005927085876464844 secondsTime to load utils op: 0.0005724430084228516 seconds + 8: + 8: + 2: Time to load utils op: 0.0006036758422851562 seconds + 2: Time to load utils op: 0.0006055831909179688 seconds + 2: Time to load utils op: 0.0006401538848876953 seconds + 9: Time to load utils op: 0.273425817489624 secondsTime to load utils op: 0.27343058586120605 seconds + 9: + 9: Time to load utils op: 0.2734334468841553 secondsTime to load utils op: 0.27344274520874023 seconds + 9: +15: Time to load utils op: 0.2762289047241211 seconds +15: Time to load utils op: 0.27623438835144043 seconds +15: Time to load utils op: 0.27588963508605957 seconds +15: Time to load utils op: 0.2765016555786133 seconds +11: Time to load utils op: 0.2746431827545166 secondsTime to load utils op: 0.2746412754058838 seconds +11: +14: Time to load utils op: 0.0005390644073486328 seconds +11: Time to load utils op: 0.2746596336364746 seconds +11: Time to load utils op: 0.27466440200805664 seconds +14: Time to load utils op: 0.0005121231079101562 seconds +14: Time to load utils op: 0.0005121231079101562 seconds +14: Time to load utils op: 0.0005097389221191406 seconds +10: Time to load utils op: 0.27555203437805176 secondsTime to load utils op: 0.2755439281463623 seconds +10: +10: Time to load utils op: 0.27556610107421875 seconds +10: Time to load utils op: 0.2755706310272217 seconds +13: Time to load utils op: 0.0004942417144775391 seconds +13: Time to load utils op: 0.0004913806915283203 secondsTime to load utils op: 0.0004737377166748047 seconds +13: +13: Time to load utils op: 0.0005304813385009766 seconds +12: Time to load utils op: 0.276226282119751 secondsTime to load utils op: 0.27623558044433594 secondsTime to load utils op: 0.27623414993286133 seconds +12: +12: +12: Time to load utils op: 0.27625036239624023 seconds +17: Time to load utils op: 0.2734363079071045 secondsTime to load utils op: 0.27344250679016113 secondsTime to load utils op: 0.27344799041748047 seconds +17: +17: +17: Time to load utils op: 0.2734658718109131 seconds +21: Time to load utils op: 0.27697277069091797 secondsTime to load utils op: 0.2767627239227295 seconds +21: +21: Time to load utils op: 0.27636289596557617 seconds +21: Time to load utils op: 0.27645087242126465 seconds +18: Time to load utils op: 0.27524733543395996 secondsTime to load utils op: 0.275249719619751 seconds +18: +18: Time to load utils op: 0.2752668857574463 seconds +18: Time to load utils op: 0.2752833366394043 seconds +19: Time to load utils op: 0.2750704288482666 seconds +19: Time to load utils op: 0.27507781982421875 seconds +19: Time to load utils op: 0.27507638931274414 seconds +19: Time to load utils op: 0.27508974075317383 seconds +29: Time to load utils op: 0.2745635509490967 seconds +29: Time to load utils op: 0.2746098041534424 secondsTime to load utils op: 0.2744448184967041 seconds +29: +29: Time to load utils op: 0.2744302749633789 seconds +22: Time to load utils op: 0.2740921974182129 secondsTime to load utils op: 0.2740952968597412 seconds +22: +22: Time to load utils op: 0.2741100788116455 seconds +22: Time to load utils op: 0.27411699295043945 seconds +26: Time to load utils op: 0.27231454849243164 seconds +26: Time to load utils op: 0.27232885360717773 secondsTime to load utils op: 0.27233290672302246 seconds +26: +26: Time to load utils op: 0.27234482765197754 seconds +25: Time to load utils op: 0.2731308937072754 seconds +25: Time to load utils op: 0.2731649875640869 seconds +25: Time to load utils op: 0.2732069492340088 seconds +25: Time to load utils op: 0.27321863174438477 seconds +24: Time to load utils op: 0.27886533737182617 secondsTime to load utils op: 0.2791121006011963 secondsTime to load utils op: 0.27932262420654297 seconds +24: +24: +24: Time to load utils op: 0.27899909019470215 seconds +31: Time to load utils op: 0.2702810764312744 seconds +31: Time to load utils op: 0.2702920436859131 secondsTime to load utils op: 0.27028751373291016 seconds +31: +31: Time to load utils op: 0.2703242301940918 seconds +20: Time to load utils op: 0.27698612213134766 secondsTime to load utils op: 0.2769930362701416 seconds +20: +23: Time to load utils op: 0.2747514247894287 secondsTime to load utils op: 0.27475905418395996 seconds +23: +23: Time to load utils op: 0.2747647762298584 seconds +27: Time to load utils op: 0.2729921340942383 secondsTime to load utils op: 0.2729959487915039 seconds +27: +27: Time to load utils op: 0.27300357818603516 seconds +27: Time to load utils op: 0.2730104923248291 seconds +20: Time to load utils op: 0.27700328826904297 secondsTime to load utils op: 0.2770075798034668 seconds +20: +23: Time to load utils op: 0.27477169036865234 seconds + 7: Time to load utils op: 0.0009267330169677734 seconds + 7: Time to load utils op: 0.0009226799011230469 secondsTime to load utils op: 0.0009131431579589844 seconds + 7: + 7: Time to load utils op: 0.0008754730224609375 seconds +28: Time to load utils op: 0.2735633850097656 seconds +28: Time to load utils op: 0.27356624603271484 seconds +28: Time to load utils op: 0.27358174324035645 seconds +28: Time to load utils op: 0.27358269691467285 seconds +16: Time to load utils op: 0.2800636291503906 seconds +16: Time to load utils op: 0.28008103370666504 seconds +16: Time to load utils op: 0.2800912857055664 seconds +16: Time to load utils op: 0.28009653091430664 seconds + 4: Time to load utils op: 0.0005037784576416016 seconds + 4: Time to load utils op: 0.0004918575286865234 seconds + 4: Time to load utils op: 0.0005090236663818359 seconds + 4: Time to load utils op: 0.00048041343688964844 seconds + 6: Time to load utils op: 0.0005171298980712891 seconds + 6: Time to load utils op: 0.0005054473876953125 secondsTime to load utils op: 0.0005271434783935547 seconds + 6: + 6: Time to load utils op: 0.0005261898040771484 seconds +30: Time to load utils op: 0.273958683013916 seconds +30: Time to load utils op: 0.27396416664123535 seconds +30: Time to load utils op: 0.2743077278137207 secondsTime to load utils op: 0.2740182876586914 seconds +30: + 3: Time to load utils op: 0.0004513263702392578 seconds + 3: Time to load utils op: 0.0005028247833251953 secondsTime to load utils op: 0.0005319118499755859 seconds + 3: + 3: Time to load utils op: 0.0005147457122802734 seconds + 0: [2023-04-29 09:45:01,809] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-04-29 09:45:01,810] [INFO] [utils.py:828:see_memory_usage] MA 4.15 GB Max_MA 4.15 GB CA 4.18 GB Max_CA 4 GB + 0: [2023-04-29 09:45:01,810] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.08 GB, percent = 8.0% +15: Time to load utils op: 0.0005168914794921875 seconds +15: Time to load utils op: 0.000553131103515625 seconds +15: Time to load utils op: 0.0005762577056884766 seconds +15: Time to load utils op: 0.0005733966827392578 seconds +11: Time to load utils op: 0.0005133152008056641 seconds +11: Time to load utils op: 0.0005168914794921875 seconds +11: Time to load utils op: 0.0005123615264892578 seconds +11: Time to load utils op: 0.0004966259002685547 seconds +29: Time to load utils op: 0.00035071372985839844 seconds +29: Time to load utils op: 0.00035119056701660156 seconds +29: Time to load utils op: 0.0003612041473388672 seconds +29: Time to load utils op: 0.00035452842712402344 seconds +24: Time to load utils op: 0.0005047321319580078 seconds +24: Time to load utils op: 0.0005218982696533203 secondsTime to load utils op: 0.0004849433898925781 seconds +24: +24: Time to load utils op: 0.0006163120269775391 seconds +17: Time to load utils op: 0.00046634674072265625 seconds +17: Time to load utils op: 0.0004994869232177734 seconds +17: Time to load utils op: 0.0005044937133789062 seconds +17: Time to load utils op: 0.0005707740783691406 seconds + 9: Time to load utils op: 0.0008244514465332031 seconds + 9: Time to load utils op: 0.0008280277252197266 seconds + 9: Time to load utils op: 0.0008568763732910156 seconds + 9: Time to load utils op: 0.0008451938629150391 seconds +12: Time to load utils op: 0.00047278404235839844 seconds +12: Time to load utils op: 0.0005021095275878906 seconds +12: Time to load utils op: 0.0005276203155517578 seconds +12: Time to load utils op: 0.0006229877471923828 seconds +19: Time to load utils op: 0.00047659873962402344 seconds +19: Time to load utils op: 0.0005166530609130859 seconds +19: Time to load utils op: 0.0005242824554443359 seconds +19: Time to load utils op: 0.0004980564117431641 seconds +10: Time to load utils op: 0.0008473396301269531 secondsTime to load utils op: 0.0008709430694580078 seconds +10: +10: Time to load utils op: 0.0008714199066162109 seconds +10: Time to load utils op: 0.0009450912475585938 seconds +21: Time to load utils op: 0.0004930496215820312 seconds +21: Time to load utils op: 0.00043272972106933594 seconds +21: Time to load utils op: 0.0004830360412597656 secondsTime to load utils op: 0.0005412101745605469 seconds +21: +25: Time to load utils op: 0.0009219646453857422 seconds +25: Time to load utils op: 0.0009129047393798828 seconds +25: Time to load utils op: 0.0008771419525146484 seconds +25: Time to load utils op: 0.0009500980377197266 seconds +31: Time to load utils op: 0.0005021095275878906 seconds +18: Time to load utils op: 0.0007905960083007812 seconds +31: Time to load utils op: 0.0005209445953369141 seconds +31: Time to load utils op: 0.0005106925964355469 secondsTime to load utils op: 0.0005307197570800781 seconds +31: +18: Time to load utils op: 0.0008747577667236328 seconds +18: Time to load utils op: 0.0008921623229980469 seconds +18: Time to load utils op: 0.0009510517120361328 seconds +16: Time to load utils op: 0.0007596015930175781 seconds +16: Time to load utils op: 0.0008234977722167969 seconds +16: Time to load utils op: 0.0008296966552734375 seconds +16: Time to load utils op: 0.0008797645568847656 seconds +22: Time to load utils op: 0.0004813671112060547 seconds +22: Time to load utils op: 0.000499725341796875 seconds +22: Time to load utils op: 0.0005254745483398438 seconds +22: Time to load utils op: 0.000545501708984375 seconds +23: Time to load utils op: 0.0005116462707519531 seconds +26: Time to load utils op: 0.0009577274322509766 seconds +23: Time to load utils op: 0.0004909038543701172 seconds +20: Time to load utils op: 0.0004930496215820312 seconds +23: Time to load utils op: 0.0005097389221191406 seconds +20: Time to load utils op: 0.0004820823669433594 seconds +23: Time to load utils op: 0.0005352497100830078 seconds +26: Time to load utils op: 0.000926971435546875 seconds +20: Time to load utils op: 0.0005273818969726562 seconds +26: Time to load utils op: 0.0009093284606933594 seconds +20: Time to load utils op: 0.0004923343658447266 seconds +26: Time to load utils op: 0.001009225845336914 seconds +28: Time to load utils op: 0.0004703998565673828 seconds +28: Time to load utils op: 0.00048160552978515625 seconds +28: Time to load utils op: 0.0005245208740234375 seconds +28: Time to load utils op: 0.0005342960357666016 seconds +27: Time to load utils op: 0.00047135353088378906 seconds +27: Time to load utils op: 0.0005078315734863281 seconds +27: Time to load utils op: 0.00043320655822753906 seconds +27: Time to load utils op: 0.0004909038543701172 seconds +30: Time to load utils op: 0.00080108642578125 seconds +30: Time to load utils op: 0.0009212493896484375 secondsTime to load utils op: 0.0008885860443115234 seconds +30: +30: Time to load utils op: 0.0008935928344726562 seconds + 0: ninja: no work to do. + 0: Time to load utils op: 0.2584545612335205 seconds + 0: [2023-04-29 09:45:02,021] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-04-29 09:45:02,021] [INFO] [utils.py:828:see_memory_usage] MA 8.44 GB Max_MA 8.44 GB CA 10.57 GB Max_CA 11 GB + 0: [2023-04-29 09:45:02,022] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.19 GB, percent = 8.0% + 0: Time to load utils op: 0.3063051700592041 seconds + 1: Time to load utils op: 0.3062553405761719 seconds + 0: Time to load utils op: 0.3081352710723877 seconds + 8: Time to load utils op: 0.3064401149749756 seconds + 2: Time to load utils op: 0.30784153938293457 seconds + 4: Time to load utils op: 0.30800318717956543 seconds + 5: Time to load utils op: 0.3085033893585205 seconds + 6: Time to load utils op: 0.30689072608947754 seconds + 7: Time to load utils op: 0.3083994388580322 seconds + 8: Time to load utils op: 0.30940890312194824 seconds + 3: Time to load utils op: 0.30805301666259766 seconds + 1: Time to load utils op: 0.3110194206237793 seconds +10: Time to load utils op: 0.3078024387359619 seconds + 9: Time to load utils op: 0.30795836448669434 seconds + 0: Time to load utils op: 0.3144717216491699 seconds +17: Time to load utils op: 0.3051009178161621 seconds +12: Time to load utils op: 0.3078899383544922 seconds +13: Time to load utils op: 0.3101060390472412 seconds + 4: Time to load utils op: 0.3137843608856201 seconds +11: Time to load utils op: 0.31005859375 seconds +15: Time to load utils op: 0.30778932571411133 seconds +14: Time to load utils op: 0.30933499336242676 seconds +16: Time to load utils op: 0.30800604820251465 seconds + 5: Time to load utils op: 0.314502477645874 seconds +18: Time to load utils op: 0.30701589584350586 seconds +21: Time to load utils op: 0.30649733543395996 seconds + 6: Time to load utils op: 0.3125457763671875 seconds +19: Time to load utils op: 0.3077397346496582 seconds + 7: Time to load utils op: 0.3144054412841797 seconds + 8: Time to load utils op: 0.31339550018310547 seconds +23: Time to load utils op: 0.3063211441040039 seconds + 2: Time to load utils op: 0.31403636932373047 seconds + 3: Time to load utils op: 0.3138558864593506 seconds + 9: Time to load utils op: 0.31414055824279785 seconds + 1: Time to load utils op: 0.3183596134185791 seconds +10: Time to load utils op: 0.31386828422546387 seconds +20: Time to load utils op: 0.308758020401001 seconds +24: Time to load utils op: 0.30681633949279785 seconds +17: Time to load utils op: 0.3115391731262207 seconds +22: Time to load utils op: 0.30741119384765625 seconds +12: Time to load utils op: 0.3140418529510498 seconds +25: Time to load utils op: 0.3077125549316406 seconds +11: Time to load utils op: 0.3157463073730469 seconds + 5: Time to load utils op: 0.3197910785675049 seconds +13: Time to load utils op: 0.31577229499816895 seconds +27: Time to load utils op: 0.3074471950531006 seconds + 4: Time to load utils op: 0.3198535442352295 seconds +16: Time to load utils op: 0.313983678817749 seconds +26: Time to load utils op: 0.3084831237792969 seconds +18: Time to load utils op: 0.3130326271057129 seconds +20: Time to load utils op: 0.3122570514678955 seconds +28: Time to load utils op: 0.30765676498413086 seconds +29: Time to load utils op: 0.30832672119140625 seconds +21: Time to load utils op: 0.31319618225097656 seconds + 6: Time to load utils op: 0.3189280033111572 seconds +15: Time to load utils op: 0.3136634826660156 seconds +14: Time to load utils op: 0.31546950340270996 seconds +30: Time to load utils op: 0.3066277503967285 seconds + 2: Time to load utils op: 0.32048535346984863 seconds + 3: Time to load utils op: 0.3204479217529297 seconds + 7: Time to load utils op: 0.3204691410064697 seconds +19: Time to load utils op: 0.3145010471343994 seconds + 8: Time to load utils op: 0.32030630111694336 seconds +23: Time to load utils op: 0.31266355514526367 seconds +10: Time to load utils op: 0.3201284408569336 seconds + 9: Time to load utils op: 0.32027220726013184 seconds +24: Time to load utils op: 0.31305766105651855 seconds +16: Time to load utils op: 0.3177073001861572 seconds +31: Time to load utils op: 0.30719971656799316 seconds +17: Time to load utils op: 0.31725096702575684 seconds +12: Time to load utils op: 0.3201146125793457 seconds +22: Time to load utils op: 0.31312012672424316 seconds + 1: Time to load utils op: 0.3249471187591553 seconds +15: Time to load utils op: 0.3202321529388428 seconds +30: Time to load utils op: 0.3103957176208496 seconds + 2: Time to load utils op: 0.3231050968170166 seconds +25: Time to load utils op: 0.3139371871948242 seconds +13: Time to load utils op: 0.3214418888092041 seconds +27: Time to load utils op: 0.3132655620574951 seconds +11: Time to load utils op: 0.32185864448547363 seconds + 4: Time to load utils op: 0.3261425495147705 seconds + 5: Time to load utils op: 0.3263537883758545 seconds +18: Time to load utils op: 0.31931447982788086 seconds +20: Time to load utils op: 0.3181116580963135 seconds +26: Time to load utils op: 0.3148043155670166 seconds +29: Time to load utils op: 0.30991411209106445 seconds +28: Time to load utils op: 0.3137054443359375 seconds +21: Time to load utils op: 0.3188202381134033 seconds + 6: Time to load utils op: 0.32480645179748535 seconds +14: Time to load utils op: 0.32068824768066406 seconds + 3: Time to load utils op: 0.3261067867279053 seconds +19: Time to load utils op: 0.3202683925628662 seconds +30: Time to load utils op: 0.3132338523864746 seconds + 7: Time to load utils op: 0.3265950679779053 seconds +11: Time to load utils op: 0.3251059055328369 seconds +10: Time to load utils op: 0.32599520683288574 seconds +23: Time to load utils op: 0.31908249855041504 seconds +24: Time to load utils op: 0.3197953701019287 seconds + 9: Time to load utils op: 0.3262612819671631 seconds +20: Time to load utils op: 0.3210744857788086 seconds +16: Time to load utils op: 0.32369279861450195 seconds +25: Time to load utils op: 0.3188059329986572 seconds +31: Time to load utils op: 0.31365537643432617 seconds +26: Time to load utils op: 0.3182816505432129 seconds +12: Time to load utils op: 0.3260986804962158 seconds +17: Time to load utils op: 0.3233318328857422 seconds +22: Time to load utils op: 0.3193318843841553 seconds +13: Time to load utils op: 0.32601094245910645 seconds +15: Time to load utils op: 0.32605648040771484 seconds +19: Time to load utils op: 0.32337117195129395 seconds +27: Time to load utils op: 0.31955909729003906 seconds +18: Time to load utils op: 0.32528114318847656 seconds +28: Time to load utils op: 0.31961607933044434 seconds +25: Time to load utils op: 0.32141804695129395 seconds +29: Time to load utils op: 0.31974077224731445 seconds +21: Time to load utils op: 0.3249244689941406 seconds +22: Time to load utils op: 0.32285094261169434 seconds +14: Time to load utils op: 0.327944278717041 seconds +30: Time to load utils op: 0.31951165199279785 seconds +23: Time to load utils op: 0.3249542713165283 seconds +24: Time to load utils op: 0.32593679428100586 seconds +29: Time to load utils op: 0.32333803176879883 seconds +31: Time to load utils op: 0.3196563720703125 seconds +26: Time to load utils op: 0.32470154762268066 seconds +27: Time to load utils op: 0.3258399963378906 seconds +28: Time to load utils op: 0.3259313106536865 seconds +31: Time to load utils op: 0.3257153034210205 seconds + 0: [2023-04-29 09:45:02,132] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-04-29 09:45:02,133] [INFO] [utils.py:828:see_memory_usage] MA 8.44 GB Max_MA 8.44 GB CA 10.57 GB Max_CA 11 GB + 0: [2023-04-29 09:45:02,133] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.24 GB, percent = 8.0% + 5: Time to load utils op: 0.0019676685333251953 seconds + 5: Time to load utils op: 0.0018231868743896484 secondsTime to load utils op: 0.0018379688262939453 seconds + 5: + 5: Time to load utils op: 0.002000570297241211 seconds +25: Time to load utils op: 0.005166053771972656 secondsTime to load utils op: 0.005247831344604492 seconds +25: +25: Time to load utils op: 0.005086183547973633 seconds +25: Time to load utils op: 0.0003409385681152344 seconds + 0: [2023-04-29 09:45:02,244] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-04-29 09:45:02,245] [INFO] [utils.py:828:see_memory_usage] MA 12.5 GB Max_MA 12.5 GB CA 16.62 GB Max_CA 17 GB + 0: [2023-04-29 09:45:02,245] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.29 GB, percent = 8.0% + 0: Time to load utils op: 0.0052831172943115234 seconds + 8: Time to load utils op: 0.0033957958221435547 secondsTime to load utils op: 0.0036711692810058594 secondsTime to load utils op: 0.003607034683227539 seconds + 8: + 8: + 8: Time to load utils op: 0.006596088409423828 seconds +16: Time to load utils op: 0.004176139831542969 seconds +16: Time to load utils op: 0.004212379455566406 seconds +16: Time to load utils op: 0.0004668235778808594 seconds +16: Time to load utils op: 0.00039887428283691406 seconds +10: Time to load utils op: 0.0036072731018066406 seconds +10: Time to load utils op: 0.006681680679321289 seconds +10: Time to load utils op: 0.006665468215942383 seconds +10: Time to load utils op: 0.0067331790924072266 seconds + 0: [2023-04-29 09:45:02,353] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-04-29 09:45:02,353] [INFO] [utils.py:828:see_memory_usage] MA 12.5 GB Max_MA 12.5 GB CA 16.62 GB Max_CA 17 GB + 0: [2023-04-29 09:45:02,353] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.34 GB, percent = 8.0% + 2: Time to load utils op: 0.003979206085205078 seconds + 2: Time to load utils op: 0.0041980743408203125 seconds + 2: Time to load utils op: 0.0004627704620361328 secondsTime to load utils op: 0.0004642009735107422 seconds + 2: + 0: Time to load utils op: 0.000400543212890625 seconds + 0: Time to load utils op: 0.00039577484130859375 seconds + 0: Time to load utils op: 0.0003933906555175781 seconds +21: Time to load utils op: 0.0044879913330078125 seconds +21: Time to load utils op: 0.004056453704833984 seconds +21: Time to load utils op: 0.000377655029296875 seconds +21: Time to load utils op: 0.0004901885986328125 seconds + 3: Time to load utils op: 0.0039010047912597656 seconds +23: Time to load utils op: 0.004424333572387695 seconds +23: Time to load utils op: 0.0003733634948730469 seconds +30: Time to load utils op: 0.004312038421630859 seconds +15: Time to load utils op: 0.0038645267486572266 seconds +23: Time to load utils op: 0.0004737377166748047 seconds +15: Time to load utils op: 0.0004239082336425781 seconds +30: Time to load utils op: 0.0004837512969970703 seconds +30: Time to load utils op: 0.00045108795166015625 secondsTime to load utils op: 0.0004456043243408203 seconds +30: +22: Time to load utils op: 0.0039081573486328125 seconds +22: Time to load utils op: 0.0041277408599853516 seconds +27: Time to load utils op: 0.00451350212097168 seconds +27: Time to load utils op: 0.00036644935607910156 seconds +24: Time to load utils op: 0.0037856101989746094 seconds +29: Time to load utils op: 0.004061222076416016 seconds +13: Time to load utils op: 0.003952503204345703 seconds + 0: [2023-04-29 09:45:02,466] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-04-29 09:45:02,467] [INFO] [utils.py:828:see_memory_usage] MA 12.51 GB Max_MA 12.51 GB CA 16.62 GB Max_CA 17 GB + 0: [2023-04-29 09:45:02,467] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.44 GB, percent = 8.0% +12: Time to load utils op: 0.0036895275115966797 seconds +29: Time to load utils op: 0.00045490264892578125 secondsTime to load utils op: 0.0004525184631347656 seconds +29: +12: Time to load utils op: 0.0033507347106933594 secondsTime to load utils op: 0.003438234329223633 secondsTime to load utils op: 0.0034301280975341797 seconds +12: +12: +24: Time to load utils op: 0.00438690185546875 seconds +13: Time to load utils op: 0.0004856586456298828 seconds +24: Time to load utils op: 0.0003485679626464844 seconds +27: Time to load utils op: 0.0004801750183105469 seconds +22: Time to load utils op: 0.0005366802215576172 seconds +24: Time to load utils op: 0.00048279762268066406 seconds +28: Time to load utils op: 0.003908634185791016 secondsTime to load utils op: 0.003918886184692383 seconds +28: + 3: Time to load utils op: 0.00043463706970214844 seconds + 6: Time to load utils op: 0.003899097442626953 seconds + 6: Time to load utils op: 0.003962039947509766 seconds +27: Time to load utils op: 0.00046563148498535156 seconds +22: Time to load utils op: 0.0004470348358154297 seconds +28: Time to load utils op: 0.00036215782165527344 seconds +17: Time to load utils op: 0.004163026809692383 seconds +19: Time to load utils op: 0.004492282867431641 seconds +15: Time to load utils op: 0.0005307197570800781 seconds + 1: Time to load utils op: 0.004137754440307617 seconds +18: Time to load utils op: 0.0038933753967285156 seconds +23: Time to load utils op: 0.0004718303680419922 seconds +17: Time to load utils op: 0.00048422813415527344 seconds +28: Time to load utils op: 0.0004291534423828125 seconds +18: Time to load utils op: 0.0004687309265136719 seconds + 7: Time to load utils op: 0.00437617301940918 seconds +19: Time to load utils op: 0.00045561790466308594 seconds +14: Time to load utils op: 0.004125833511352539 seconds + 1: Time to load utils op: 0.004413127899169922 seconds + 3: Time to load utils op: 0.0004811286926269531 seconds + 4: Time to load utils op: 0.0038824081420898438 seconds +11: Time to load utils op: 0.004148244857788086 seconds +29: Time to load utils op: 0.00039076805114746094 seconds +17: Time to load utils op: 0.00040531158447265625 seconds +13: Time to load utils op: 0.0004830360412597656 seconds +19: Time to load utils op: 0.00039124488830566406 seconds +11: Time to load utils op: 0.0004296302795410156 seconds + 4: Time to load utils op: 0.00429081916809082 seconds + 4: Time to load utils op: 0.004091024398803711 seconds + 6: Time to load utils op: 0.000476837158203125 seconds +26: Time to load utils op: 0.004503011703491211 seconds +15: Time to load utils op: 0.0004813671112060547 seconds +14: Time to load utils op: 0.00048661231994628906 seconds + 1: Time to load utils op: 0.0004856586456298828 seconds +19: Time to load utils op: 0.0004642009735107422 seconds +20: Time to load utils op: 0.004446983337402344 seconds + 9: Time to load utils op: 0.003959178924560547 seconds + 9: Time to load utils op: 0.0039980411529541016 seconds + 6: Time to load utils op: 0.0004711151123046875 seconds +26: Time to load utils op: 0.003958225250244141 seconds + 3: Time to load utils op: 0.0004088878631591797 seconds +26: Time to load utils op: 0.00047969818115234375 seconds +13: Time to load utils op: 0.0004131793975830078 seconds +20: Time to load utils op: 0.00048661231994628906 seconds +20: Time to load utils op: 0.00046896934509277344 seconds + 9: Time to load utils op: 0.0004744529724121094 seconds + 9: Time to load utils op: 0.00048828125 seconds +11: Time to load utils op: 0.0004773139953613281 seconds + 7: Time to load utils op: 0.00047397613525390625 seconds + 7: Time to load utils op: 0.0004417896270751953 seconds +14: Time to load utils op: 0.00042366981506347656 seconds +20: Time to load utils op: 0.0004203319549560547 seconds + 4: Time to load utils op: 0.0005059242248535156 seconds +11: Time to load utils op: 0.00047016143798828125 seconds + 7: Time to load utils op: 0.000400543212890625 seconds +26: Time to load utils op: 0.0004863739013671875 seconds +31: Time to load utils op: 0.004565238952636719 seconds +31: Time to load utils op: 0.00048732757568359375 seconds +17: Time to load utils op: 0.0004925727844238281 seconds + 1: Time to load utils op: 0.00045800209045410156 seconds +18: Time to load utils op: 0.0004558563232421875 secondsTime to load utils op: 0.00046515464782714844 seconds +18: +31: Time to load utils op: 0.0004169940948486328 seconds +31: Time to load utils op: 0.0004794597625732422 seconds +14: Time to load utils op: 0.00047278404235839844 seconds + 0: [2023-04-29 09:45:02,590] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-04-29 09:45:02,590] [INFO] [utils.py:828:see_memory_usage] MA 12.51 GB Max_MA 12.51 GB CA 16.62 GB Max_CA 17 GB + 0: [2023-04-29 09:45:02,590] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.64 GB, percent = 8.1% + 0: [2023-04-29 09:45:02,732] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-04-29 09:45:02,732] [INFO] [utils.py:828:see_memory_usage] MA 12.64 GB Max_MA 12.64 GB CA 16.62 GB Max_CA 17 GB + 0: [2023-04-29 09:45:02,732] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.78 GB, percent = 8.1% + 0: [2023-04-29 09:45:02,844] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-04-29 09:45:02,844] [INFO] [utils.py:828:see_memory_usage] MA 12.64 GB Max_MA 12.64 GB CA 16.62 GB Max_CA 17 GB + 0: [2023-04-29 09:45:02,844] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.73 GB, percent = 8.1% + 0: [2023-04-29 09:45:02,845] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-04-29 09:45:02,845] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-04-29 09:45:02,845] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-04-29 09:45:02,845] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-04-29 09:45:02,845] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-04-29 09:45:02,846] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] train_batch_size ............. 512 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 1 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-04-29 09:45:02,847] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-04-29 09:45:02,848] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-04-29 09:45:02,848] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-04-29 09:45:02,848] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-04-29 09:45:02,848] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 1, + 0: "train_batch_size": 512, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0004420280456542969 seconds + 0: [2023-04-29 09:45:02,848] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=1 + 0: [2023-04-29 09:45:03,005] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=24 [0, 24) STAGE_PARAMS=2226419712 (2226.420M) TOTAL_PARAMS=8905695232 (8905.695M) UNIQUE_PARAMS=8682348544 (8682.349M) +32: [2023-04-29 09:45:03,005] [INFO] [engine.py:145:__init__] RANK=257 STAGE=1 LAYERS=25 [24, 49) STAGE_PARAMS=2226427904 (2226.428M) TOTAL_PARAMS=8905695232 (8905.695M) UNIQUE_PARAMS=8682348544 (8682.349M) +32: [2023-04-29 09:45:03,005] [INFO] [engine.py:145:__init__] RANK=256 STAGE=1 LAYERS=25 [24, 49) STAGE_PARAMS=2226427904 (2226.428M) TOTAL_PARAMS=8905695232 (8905.695M) UNIQUE_PARAMS=8682348544 (8682.349M) + 0: [2023-04-29 09:45:03,006] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=24 [0, 24) STAGE_PARAMS=2226419712 (2226.420M) TOTAL_PARAMS=8905695232 (8905.695M) UNIQUE_PARAMS=8682348544 (8682.349M) +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +56: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +63: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +55: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +54: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +34: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +61: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +48: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +52: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +35: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +42: [2023-04-29 09:45:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +40: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +41: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +47: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +44: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +38: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +53: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +59: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +57: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +49: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +39: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +33: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +45: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 0: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 0: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 0: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 0: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 0: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 0: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +32: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 0: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +61: [2023-04-29 09:45:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +61: [2023-04-29 09:45:03,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +16: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +16: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +16: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +16: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +63: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +48: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +56: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +16: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 1: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +16: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +16: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +16: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +54: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +43: [2023-04-29 09:45:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +43: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +43: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +43: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +43: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +43: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +16: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +54: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 1: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 1: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +62: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 1: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 1: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 1: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +34: [2023-04-29 09:45:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 8: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +42: [2023-04-29 09:45:03,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 8: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +42: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 8: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 4: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +27: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 4: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 4: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +58: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 4: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +27: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +27: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 4: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +27: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +27: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +20: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 4: [2023-04-29 09:45:03,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +20: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +27: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 4: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +41: [2023-04-29 09:45:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +24: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +60: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +55: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +24: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +40: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 9: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +22: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +14: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 5: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 9: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 9: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +30: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 9: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +24: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +40: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +24: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +14: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +15: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +24: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +12: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +33: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +15: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +19: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 5: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +30: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +19: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +19: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +19: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +30: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 9: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +30: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +52: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +12: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +15: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +12: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +14: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +14: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +22: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +12: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +44: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +47: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 9: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +13: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +15: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +35: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +19: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 9: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +22: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +49: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +30: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +19: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 9: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +44: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +30: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +12: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +49: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +30: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +29: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +29: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +29: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +12: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +37: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +19: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +19: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +46: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +23: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +10: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +13: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +19: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +29: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +51: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +19: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +28: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +26: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +38: [2023-04-29 09:45:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +37: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +29: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +13: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +26: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +10: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +28: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +11: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +23: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +25: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +28: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +10: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +26: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +29: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +21: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +21: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +21: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +11: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +29: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +26: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +23: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +11: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +32: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +32: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +26: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +28: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +32: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +10: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +25: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +53: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +21: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +26: [2023-04-29 09:45:03,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +28: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +21: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +32: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +25: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 0: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +11: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +25: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt... +32: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 6: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +11: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 0: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 7: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +45: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +32: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +17: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +21: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +32: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 3: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +11: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +18: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +50: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +32: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... + 2: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +57: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +31: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +36: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +11: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +59: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +25: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +39: [2023-04-29 09:45:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt... +21: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 0: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 7: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +18: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 6: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 2: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. + 3: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +17: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_00_model_states.pt. +31: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +63: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +63: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +63: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +63: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +56: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +56: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +56: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +56: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +58: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +58: [2023-04-29 09:45:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +43: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +43: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +48: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +54: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +54: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +54: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +52: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +52: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +52: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +58: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +62: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +60: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +60: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +60: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +34: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +44: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +63: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +41: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +41: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +43: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +16: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +48: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +42: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +42: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +42: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +49: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +35: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +35: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +35: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +54: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +58: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +62: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +62: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +60: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +34: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +40: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +48: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +42: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. + 1: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +61: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +51: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +51: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +62: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +43: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +16: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +16: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +38: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +38: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +49: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +49: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +33: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +33: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +33: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +55: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +55: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +63: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +39: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +41: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +34: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +40: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +48: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +44: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +44: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +50: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +47: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +47: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. + 1: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 1: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +53: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +34: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +16: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +38: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +47: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +45: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +45: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +45: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +33: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +63: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +40: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +50: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +50: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. + 4: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +35: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +61: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +55: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +39: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +36: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +36: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +44: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +47: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +45: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. + 1: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +56: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +14: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +20: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +20: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +20: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +38: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +40: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +46: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +46: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +46: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +50: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. + 4: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 4: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 5: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 5: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +53: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +59: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +57: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +57: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +27: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +39: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +24: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +49: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +59: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +58: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +14: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +14: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +36: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. + 4: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 5: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +59: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +57: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +43: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +16: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +56: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 8: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +27: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +48: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 5: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +60: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +14: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 1: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +52: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +43: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +20: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +53: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +59: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +62: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +41: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +44: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +52: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +10: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +10: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +58: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +62: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +27: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +16: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +48: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 1: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 1: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +35: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +54: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +13: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +43: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +16: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +38: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +26: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +44: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +52: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +62: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +21: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +27: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +34: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +16: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +28: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +28: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +42: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +13: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +23: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +23: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +23: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +25: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +25: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +24: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +26: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +26: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +48: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 1: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +51: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +10: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +62: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +13: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +21: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +21: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +41: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +28: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +40: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +42: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +39: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +26: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +48: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +47: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +31: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +13: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +23: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +21: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +25: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +27: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 7: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 7: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +39: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +28: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +20: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +38: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +42: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 2: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 5: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 5: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +49: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +10: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 8: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +31: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +25: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +17: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +27: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +24: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +40: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 6: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 6: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +50: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +42: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 3: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. + 3: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +47: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 5: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +45: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +17: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +17: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +39: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +14: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +14: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 4: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +47: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 5: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +49: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +40: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +45: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +39: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +18: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +14: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +57: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 8: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +12: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +24: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +26: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +36: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_01_model_states.pt. +57: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 9: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +14: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +21: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +12: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +26: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +32: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +32: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +32: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +26: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +30: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +26: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +32: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. + 6: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +32: [2023-04-29 09:45:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:03,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +61: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +61: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +61: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +41: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +41: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +43: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +34: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +56: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +56: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +56: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +41: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +34: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +34: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +61: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +54: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +54: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +60: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +60: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +60: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +43: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +42: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +42: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +56: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +43: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +52: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +52: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +52: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +58: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +58: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +58: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +34: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +42: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +55: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +55: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +55: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +43: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +54: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +60: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +41: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +33: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +33: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +33: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +47: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +47: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +54: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +47: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +55: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +58: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +42: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +45: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +44: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +38: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +38: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +47: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +33: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +51: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +51: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +39: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +44: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +44: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +45: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +45: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +51: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +35: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +35: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +39: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +48: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +49: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +49: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +38: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +35: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +59: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +44: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +45: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +51: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +48: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +61: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +61: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +61: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +41: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +43: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +49: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +35: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +34: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +54: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +40: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +61: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +62: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +63: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +42: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +42: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +50: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +50: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +55: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +42: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +62: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +33: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +42: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +59: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +37: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +37: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +37: [2023-04-29 09:45:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_03_model_states.pt. +37: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +37: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +37: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +37: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/mp_rank_02_model_states.pt. +37: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +32: [2023-04-29 09:45:04,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:04,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +32: [2023-04-29 09:45:04,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:04,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +32: [2023-04-29 09:45:04,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +32: [2023-04-29 09:45:04,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +32: [2023-04-29 09:45:04,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +61: [2023-04-29 09:45:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +61: [2023-04-29 09:45:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +61: [2023-04-29 09:45:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +61: [2023-04-29 09:45:04,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +61: [2023-04-29 09:45:04,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +61: [2023-04-29 09:45:04,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +61: [2023-04-29 09:45:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +32: [2023-04-29 09:45:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:04,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +51: [2023-04-29 09:45:04,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +51: [2023-04-29 09:45:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +51: [2023-04-29 09:45:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +51: [2023-04-29 09:45:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +51: [2023-04-29 09:45:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:04,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +38: [2023-04-29 09:45:04,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +51: [2023-04-29 09:45:04,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +54: [2023-04-29 09:45:04,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +54: [2023-04-29 09:45:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +38: [2023-04-29 09:45:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +38: [2023-04-29 09:45:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +54: [2023-04-29 09:45:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +54: [2023-04-29 09:45:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +54: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:04,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +52: [2023-04-29 09:45:04,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +52: [2023-04-29 09:45:04,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +52: [2023-04-29 09:45:04,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +52: [2023-04-29 09:45:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:04,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:04,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +63: [2023-04-29 09:45:04,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +63: [2023-04-29 09:45:04,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:04,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +63: [2023-04-29 09:45:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +62: [2023-04-29 09:45:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +62: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +62: [2023-04-29 09:45:04,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +62: [2023-04-29 09:45:04,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +61: [2023-04-29 09:45:04,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +62: [2023-04-29 09:45:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +62: [2023-04-29 09:45:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +20: [2023-04-29 09:45:04,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 4: [2023-04-29 09:45:04,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 4: [2023-04-29 09:45:04,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 4: [2023-04-29 09:45:04,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +20: [2023-04-29 09:45:04,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 4: [2023-04-29 09:45:04,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +61: [2023-04-29 09:45:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +20: [2023-04-29 09:45:04,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +20: [2023-04-29 09:45:04,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +20: [2023-04-29 09:45:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +20: [2023-04-29 09:45:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +20: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +34: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +34: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +34: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +10: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +34: [2023-04-29 09:45:04,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +34: [2023-04-29 09:45:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +34: [2023-04-29 09:45:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +16: [2023-04-29 09:45:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +61: [2023-04-29 09:45:04,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +16: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +38: [2023-04-29 09:45:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:04,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +51: [2023-04-29 09:45:04,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:04,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +41: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +39: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +41: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +41: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +39: [2023-04-29 09:45:04,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +39: [2023-04-29 09:45:04,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +16: [2023-04-29 09:45:04,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +16: [2023-04-29 09:45:04,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +43: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +16: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +41: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +53: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +10: [2023-04-29 09:45:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +10: [2023-04-29 09:45:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +10: [2023-04-29 09:45:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +10: [2023-04-29 09:45:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +26: [2023-04-29 09:45:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +26: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +61: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +26: [2023-04-29 09:45:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +43: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +43: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +38: [2023-04-29 09:45:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +16: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +38: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +16: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +59: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +16: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +16: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +16: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +45: [2023-04-29 09:45:04,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +46: [2023-04-29 09:45:04,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +46: [2023-04-29 09:45:04,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +26: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +26: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +56: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +26: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +26: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 2: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +24: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +56: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 2: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +56: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +54: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +26: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +24: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +60: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +33: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +60: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +26: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +60: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +33: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +52: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +52: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +26: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +53: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +53: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +33: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +53: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +45: [2023-04-29 09:45:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +51: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +45: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +53: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 2: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +57: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +17: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +36: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +36: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +57: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +59: [2023-04-29 09:45:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +36: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +59: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +59: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 2: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +36: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +12: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +60: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +48: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +56: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +17: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +45: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +60: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +17: [2023-04-29 09:45:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +23: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +60: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +59: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +24: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +24: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +56: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +36: [2023-04-29 09:45:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +36: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +36: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +36: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +57: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +57: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +57: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +57: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +57: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +63: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +57: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +51: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +40: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +40: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +40: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +40: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +24: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +25: [2023-04-29 09:45:04,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +56: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +25: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +25: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +25: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +25: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +25: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +51: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +25: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +51: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +48: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +40: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +17: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +12: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +63: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +63: [2023-04-29 09:45:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +12: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +12: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +12: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +23: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +12: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +24: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +12: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +24: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +17: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +48: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +22: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +48: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 1: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +23: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +18: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +48: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +23: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +48: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +24: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +23: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +21: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +23: [2023-04-29 09:45:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 3: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +21: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +21: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +17: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +21: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +17: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +21: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +52: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +52: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +22: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 1: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +22: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +19: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +31: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +54: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +22: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +28: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +54: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 0: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +25: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +25: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +48: [2023-04-29 09:45:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +25: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 1: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +25: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +48: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 1: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +48: [2023-04-29 09:45:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +48: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +38: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +18: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +29: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +29: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +22: [2023-04-29 09:45:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 3: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 3: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 3: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +22: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +52: [2023-04-29 09:45:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 1: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +25: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +38: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +25: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 1: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +62: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +18: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +28: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +54: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +28: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +29: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +54: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +29: [2023-04-29 09:45:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 4: [2023-04-29 09:45:04,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 3: [2023-04-29 09:45:04,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:04,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +18: [2023-04-29 09:45:04,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +18: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +29: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +31: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +31: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +11: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 6: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +31: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +31: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 6: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +37: [2023-04-29 09:45:04,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +31: [2023-04-29 09:45:04,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:04,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +31: [2023-04-29 09:45:04,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +20: [2023-04-29 09:45:04,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +63: [2023-04-29 09:45:04,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +63: [2023-04-29 09:45:04,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:04,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +28: [2023-04-29 09:45:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +28: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +28: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +28: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +28: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +62: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +19: [2023-04-29 09:45:04,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +37: [2023-04-29 09:45:04,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +34: [2023-04-29 09:45:04,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +37: [2023-04-29 09:45:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +37: [2023-04-29 09:45:04,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +19: [2023-04-29 09:45:04,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +37: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +37: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +37: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 9: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 9: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +37: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +49: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +49: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +49: [2023-04-29 09:45:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 9: [2023-04-29 09:45:04,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 9: [2023-04-29 09:45:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 9: [2023-04-29 09:45:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +39: [2023-04-29 09:45:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +20: [2023-04-29 09:45:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +20: [2023-04-29 09:45:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 5: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 5: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +49: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +49: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +49: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 5: [2023-04-29 09:45:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 5: [2023-04-29 09:45:04,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +20: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +41: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +27: [2023-04-29 09:45:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +20: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +39: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +39: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +39: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +16: [2023-04-29 09:45:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +49: [2023-04-29 09:45:04,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +14: [2023-04-29 09:45:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +46: [2023-04-29 09:45:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +27: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +49: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +47: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +43: [2023-04-29 09:45:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +27: [2023-04-29 09:45:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +27: [2023-04-29 09:45:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +41: [2023-04-29 09:45:04,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +27: [2023-04-29 09:45:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +55: [2023-04-29 09:45:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +55: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +13: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +10: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +34: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +13: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 8: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +62: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +13: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +13: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +27: [2023-04-29 09:45:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +13: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +14: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +27: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +14: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 8: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +13: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +45: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +13: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +60: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +27: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +47: [2023-04-29 09:45:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +10: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +43: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +47: [2023-04-29 09:45:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +47: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +53: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +39: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 2: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +44: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +34: [2023-04-29 09:45:04,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:04,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +53: [2023-04-29 09:45:04,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +60: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +16: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +15: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +14: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +40: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +26: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +20: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +20: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +15: [2023-04-29 09:45:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +33: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +17: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +34: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +16: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +14: [2023-04-29 09:45:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +24: [2023-04-29 09:45:04,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +39: [2023-04-29 09:45:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +14: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +44: [2023-04-29 09:45:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +44: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +26: [2023-04-29 09:45:04,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +36: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +33: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +14: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +36: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +44: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +44: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +12: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +60: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +60: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +26: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +57: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +57: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +41: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +30: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +45: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +56: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +39: [2023-04-29 09:45:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +46: [2023-04-29 09:45:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +17: [2023-04-29 09:45:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +59: [2023-04-29 09:45:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +48: [2023-04-29 09:45:04,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +59: [2023-04-29 09:45:04,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +41: [2023-04-29 09:45:04,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +25: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +43: [2023-04-29 09:45:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +60: [2023-04-29 09:45:04,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +25: [2023-04-29 09:45:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +35: [2023-04-29 09:45:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +46: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +35: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +35: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +35: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +29: [2023-04-29 09:45:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +45: [2023-04-29 09:45:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +18: [2023-04-29 09:45:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +46: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +17: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +17: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +46: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +43: [2023-04-29 09:45:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +23: [2023-04-29 09:45:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +22: [2023-04-29 09:45:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +48: [2023-04-29 09:45:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +21: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +33: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +30: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +53: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +43: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +28: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +50: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +53: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +36: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +31: [2023-04-29 09:45:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +11: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +58: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +58: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +53: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +24: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +35: [2023-04-29 09:45:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +30: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +35: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +53: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +35: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +18: [2023-04-29 09:45:04,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +59: [2023-04-29 09:45:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +60: [2023-04-29 09:45:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +56: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +25: [2023-04-29 09:45:04,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +58: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +28: [2023-04-29 09:45:04,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +58: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +58: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +40: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +24: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +35: [2023-04-29 09:45:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 2: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +35: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... +22: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +33: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +22: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +33: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 7: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 7: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt... + 7: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +50: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +50: [2023-04-29 09:45:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt... +25: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +16: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +11: [2023-04-29 09:45:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +50: [2023-04-29 09:45:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +18: [2023-04-29 09:45:04,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +18: [2023-04-29 09:45:04,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +36: [2023-04-29 09:45:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +29: [2023-04-29 09:45:04,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +10: [2023-04-29 09:45:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +36: [2023-04-29 09:45:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +26: [2023-04-29 09:45:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:04,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +26: [2023-04-29 09:45:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +58: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +31: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +58: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 6: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +60: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +33: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +16: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +37: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 6: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +45: [2023-04-29 09:45:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +58: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +29: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +59: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +16: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +50: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +50: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +45: [2023-04-29 09:45:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +10: [2023-04-29 09:45:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +50: [2023-04-29 09:45:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +58: [2023-04-29 09:45:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +10: [2023-04-29 09:45:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +60: [2023-04-29 09:45:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +50: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +48: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +58: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +26: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +57: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +57: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +25: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +45: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +56: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +59: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +22: [2023-04-29 09:45:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +40: [2023-04-29 09:45:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +56: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +12: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 0: [2023-04-29 09:45:04,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:04,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +28: [2023-04-29 09:45:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +48: [2023-04-29 09:45:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +23: [2023-04-29 09:45:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:04,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:04,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:04,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +23: [2023-04-29 09:45:04,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +48: [2023-04-29 09:45:04,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:04,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +24: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +37: [2023-04-29 09:45:04,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +37: [2023-04-29 09:45:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 1: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +21: [2023-04-29 09:45:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +28: [2023-04-29 09:45:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +22: [2023-04-29 09:45:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 2: [2023-04-29 09:45:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +42: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +42: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +37: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +18: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:04,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +47: [2023-04-29 09:45:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +31: [2023-04-29 09:45:04,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +31: [2023-04-29 09:45:04,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +29: [2023-04-29 09:45:04,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +31: [2023-04-29 09:45:04,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 0: [2023-04-29 09:45:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +42: [2023-04-29 09:45:04,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt... +42: [2023-04-29 09:45:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... +42: [2023-04-29 09:45:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt... + 0: [2023-04-29 09:45:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:04,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +47: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +49: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +55: [2023-04-29 09:45:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 3: [2023-04-29 09:45:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +27: [2023-04-29 09:45:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +13: [2023-04-29 09:45:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +15: [2023-04-29 09:45:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 9: [2023-04-29 09:45:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +37: [2023-04-29 09:45:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +37: [2023-04-29 09:45:04,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +37: [2023-04-29 09:45:04,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:04,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +19: [2023-04-29 09:45:04,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +19: [2023-04-29 09:45:04,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:04,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +44: [2023-04-29 09:45:04,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +47: [2023-04-29 09:45:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +14: [2023-04-29 09:45:04,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:04,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:04,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +49: [2023-04-29 09:45:04,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +27: [2023-04-29 09:45:04,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:04,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +35: [2023-04-29 09:45:04,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +47: [2023-04-29 09:45:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +47: [2023-04-29 09:45:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:04,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:04,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +55: [2023-04-29 09:45:04,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:04,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:04,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +13: [2023-04-29 09:45:04,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +50: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +13: [2023-04-29 09:45:04,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +35: [2023-04-29 09:45:04,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +15: [2023-04-29 09:45:04,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:04,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:04,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +35: [2023-04-29 09:45:04,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:04,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:04,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. + 8: [2023-04-29 09:45:04,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:04,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +30: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +58: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +58: [2023-04-29 09:45:04,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +44: [2023-04-29 09:45:04,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:04,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +50: [2023-04-29 09:45:04,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_00-model_states.pt. +14: [2023-04-29 09:45:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +58: [2023-04-29 09:45:04,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +58: [2023-04-29 09:45:04,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +14: [2023-04-29 09:45:04,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:04,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +35: [2023-04-29 09:45:04,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:04,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:04,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:04,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:04,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. + 7: [2023-04-29 09:45:04,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:04,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +50: [2023-04-29 09:45:04,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:04,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:04,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +30: [2023-04-29 09:45:04,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:04,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +42: [2023-04-29 09:45:04,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_00-model_states.pt. +30: [2023-04-29 09:45:04,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:04,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:04,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:04,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:04,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:04,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +54: [2023-04-29 09:45:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +54: [2023-04-29 09:45:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +32: [2023-04-29 09:45:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +32: [2023-04-29 09:45:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +32: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +38: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +38: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +54: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +38: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +38: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +32: [2023-04-29 09:45:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +61: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +61: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +61: [2023-04-29 09:45:04,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +61: [2023-04-29 09:45:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +51: [2023-04-29 09:45:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +51: [2023-04-29 09:45:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +51: [2023-04-29 09:45:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +51: [2023-04-29 09:45:04,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:04,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +34: [2023-04-29 09:45:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +34: [2023-04-29 09:45:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +34: [2023-04-29 09:45:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +63: [2023-04-29 09:45:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +63: [2023-04-29 09:45:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +63: [2023-04-29 09:45:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +63: [2023-04-29 09:45:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +34: [2023-04-29 09:45:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +53: [2023-04-29 09:45:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +53: [2023-04-29 09:45:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +53: [2023-04-29 09:45:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +60: [2023-04-29 09:45:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +60: [2023-04-29 09:45:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +60: [2023-04-29 09:45:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +62: [2023-04-29 09:45:04,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +62: [2023-04-29 09:45:04,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +62: [2023-04-29 09:45:04,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +60: [2023-04-29 09:45:04,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +62: [2023-04-29 09:45:04,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +53: [2023-04-29 09:45:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:04,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:04,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:04,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +43: [2023-04-29 09:45:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +43: [2023-04-29 09:45:04,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +43: [2023-04-29 09:45:04,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +60: [2023-04-29 09:45:04,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +60: [2023-04-29 09:45:04,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +60: [2023-04-29 09:45:04,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +60: [2023-04-29 09:45:04,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:04,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:04,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:04,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:04,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:04,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:04,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +40: [2023-04-29 09:45:04,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +40: [2023-04-29 09:45:04,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +40: [2023-04-29 09:45:04,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +43: [2023-04-29 09:45:04,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:04,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:04,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:04,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +32: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +32: [2023-04-29 09:45:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +32: [2023-04-29 09:45:04,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +32: [2023-04-29 09:45:04,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +54: [2023-04-29 09:45:04,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +54: [2023-04-29 09:45:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +63: [2023-04-29 09:45:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +51: [2023-04-29 09:45:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +51: [2023-04-29 09:45:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +51: [2023-04-29 09:45:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +63: [2023-04-29 09:45:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +63: [2023-04-29 09:45:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +54: [2023-04-29 09:45:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +59: [2023-04-29 09:45:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +59: [2023-04-29 09:45:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:04,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:04,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +61: [2023-04-29 09:45:04,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +61: [2023-04-29 09:45:04,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +61: [2023-04-29 09:45:04,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +61: [2023-04-29 09:45:04,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:04,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +61: [2023-04-29 09:45:04,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:04,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:04,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:04,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:04,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +49: [2023-04-29 09:45:04,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +33: [2023-04-29 09:45:04,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +33: [2023-04-29 09:45:04,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:04,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:04,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:04,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:04,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +33: [2023-04-29 09:45:04,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +33: [2023-04-29 09:45:04,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:04,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +52: [2023-04-29 09:45:04,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +52: [2023-04-29 09:45:04,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:04,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +52: [2023-04-29 09:45:04,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +39: [2023-04-29 09:45:04,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +38: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +38: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +38: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +32: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:04,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:04,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +56: [2023-04-29 09:45:04,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +56: [2023-04-29 09:45:04,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +56: [2023-04-29 09:45:04,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +56: [2023-04-29 09:45:04,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +51: [2023-04-29 09:45:04,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:04,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +49: [2023-04-29 09:45:04,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +51: [2023-04-29 09:45:04,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +32: [2023-04-29 09:45:04,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 4: [2023-04-29 09:45:04,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 4: [2023-04-29 09:45:04,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 4: [2023-04-29 09:45:04,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +33: [2023-04-29 09:45:04,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +33: [2023-04-29 09:45:04,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +63: [2023-04-29 09:45:04,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:04,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +24: [2023-04-29 09:45:04,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +24: [2023-04-29 09:45:04,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +24: [2023-04-29 09:45:04,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +24: [2023-04-29 09:45:04,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +33: [2023-04-29 09:45:04,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +54: [2023-04-29 09:45:04,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:04,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +18: [2023-04-29 09:45:04,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +18: [2023-04-29 09:45:04,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 5: [2023-04-29 09:45:04,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 5: [2023-04-29 09:45:04,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 5: [2023-04-29 09:45:04,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +54: [2023-04-29 09:45:04,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +54: [2023-04-29 09:45:04,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 5: [2023-04-29 09:45:04,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +18: [2023-04-29 09:45:04,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +33: [2023-04-29 09:45:04,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +18: [2023-04-29 09:45:04,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +29: [2023-04-29 09:45:04,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +29: [2023-04-29 09:45:04,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +29: [2023-04-29 09:45:04,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +29: [2023-04-29 09:45:04,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +20: [2023-04-29 09:45:04,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +20: [2023-04-29 09:45:04,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +48: [2023-04-29 09:45:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +48: [2023-04-29 09:45:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +45: [2023-04-29 09:45:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +45: [2023-04-29 09:45:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +45: [2023-04-29 09:45:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +45: [2023-04-29 09:45:04,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +20: [2023-04-29 09:45:04,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +51: [2023-04-29 09:45:04,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +20: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +56: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +20: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +48: [2023-04-29 09:45:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +20: [2023-04-29 09:45:04,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +61: [2023-04-29 09:45:04,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:04,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +56: [2023-04-29 09:45:04,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:04,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:04,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +32: [2023-04-29 09:45:04,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:04,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:04,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +41: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +41: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +20: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +51: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 4: [2023-04-29 09:45:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +61: [2023-04-29 09:45:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +61: [2023-04-29 09:45:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +27: [2023-04-29 09:45:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +27: [2023-04-29 09:45:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +39: [2023-04-29 09:45:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +51: [2023-04-29 09:45:04,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:04,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +37: [2023-04-29 09:45:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +32: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +27: [2023-04-29 09:45:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +24: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +27: [2023-04-29 09:45:04,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +37: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +20: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +39: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 4: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:04,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +57: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +57: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +41: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +41: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +41: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +63: [2023-04-29 09:45:04,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:04,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:04,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:04,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +34: [2023-04-29 09:45:04,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +20: [2023-04-29 09:45:04,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +39: [2023-04-29 09:45:04,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +39: [2023-04-29 09:45:04,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +24: [2023-04-29 09:45:04,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:04,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +21: [2023-04-29 09:45:04,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +18: [2023-04-29 09:45:04,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:04,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +21: [2023-04-29 09:45:04,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 5: [2023-04-29 09:45:04,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 5: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +39: [2023-04-29 09:45:04,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +34: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +34: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +34: [2023-04-29 09:45:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +24: [2023-04-29 09:45:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +34: [2023-04-29 09:45:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +29: [2023-04-29 09:45:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +34: [2023-04-29 09:45:05,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +18: [2023-04-29 09:45:05,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:05,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:05,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +29: [2023-04-29 09:45:05,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +24: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +24: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +24: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +45: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +55: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +55: [2023-04-29 09:45:05,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +55: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +54: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:05,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +42: [2023-04-29 09:45:05,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +48: [2023-04-29 09:45:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +44: [2023-04-29 09:45:05,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +54: [2023-04-29 09:45:05,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +42: [2023-04-29 09:45:05,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +42: [2023-04-29 09:45:05,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +24: [2023-04-29 09:45:05,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:05,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +61: [2023-04-29 09:45:05,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +24: [2023-04-29 09:45:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +24: [2023-04-29 09:45:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +41: [2023-04-29 09:45:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +27: [2023-04-29 09:45:05,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +52: [2023-04-29 09:45:05,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:05,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:05,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +37: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +61: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +27: [2023-04-29 09:45:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +57: [2023-04-29 09:45:05,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:05,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:05,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +36: [2023-04-29 09:45:05,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +37: [2023-04-29 09:45:05,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +27: [2023-04-29 09:45:05,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +52: [2023-04-29 09:45:05,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +52: [2023-04-29 09:45:05,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +39: [2023-04-29 09:45:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +39: [2023-04-29 09:45:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +21: [2023-04-29 09:45:05,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +36: [2023-04-29 09:45:05,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +39: [2023-04-29 09:45:05,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +21: [2023-04-29 09:45:05,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:05,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:05,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +45: [2023-04-29 09:45:05,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 0: [2023-04-29 09:45:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +39: [2023-04-29 09:45:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:05,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 0: [2023-04-29 09:45:05,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +55: [2023-04-29 09:45:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:05,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:05,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +46: [2023-04-29 09:45:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +46: [2023-04-29 09:45:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +46: [2023-04-29 09:45:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +45: [2023-04-29 09:45:05,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +38: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +38: [2023-04-29 09:45:05,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +28: [2023-04-29 09:45:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +44: [2023-04-29 09:45:05,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +12: [2023-04-29 09:45:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +12: [2023-04-29 09:45:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +31: [2023-04-29 09:45:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +31: [2023-04-29 09:45:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +31: [2023-04-29 09:45:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +31: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +17: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +17: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +12: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +17: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +12: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +17: [2023-04-29 09:45:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +35: [2023-04-29 09:45:05,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +35: [2023-04-29 09:45:05,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +36: [2023-04-29 09:45:05,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +52: [2023-04-29 09:45:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +35: [2023-04-29 09:45:05,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +52: [2023-04-29 09:45:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +39: [2023-04-29 09:45:05,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +39: [2023-04-29 09:45:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +38: [2023-04-29 09:45:05,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +46: [2023-04-29 09:45:05,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 2: [2023-04-29 09:45:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 2: [2023-04-29 09:45:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 2: [2023-04-29 09:45:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +46: [2023-04-29 09:45:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +31: [2023-04-29 09:45:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:05,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:05,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:05,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +38: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +21: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +38: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +24: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +35: [2023-04-29 09:45:05,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:05,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +23: [2023-04-29 09:45:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +23: [2023-04-29 09:45:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +23: [2023-04-29 09:45:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +24: [2023-04-29 09:45:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +17: [2023-04-29 09:45:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +14: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +14: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 3: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 3: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +14: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +14: [2023-04-29 09:45:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +35: [2023-04-29 09:45:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +17: [2023-04-29 09:45:05,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +45: [2023-04-29 09:45:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +14: [2023-04-29 09:45:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:05,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:05,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +34: [2023-04-29 09:45:05,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 8: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 8: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +41: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +62: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +62: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +24: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +11: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +14: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 7: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +14: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 7: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +28: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 9: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +14: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 9: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 9: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +11: [2023-04-29 09:45:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +25: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +25: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +25: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +20: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +45: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +45: [2023-04-29 09:45:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 4: [2023-04-29 09:45:05,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +26: [2023-04-29 09:45:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +26: [2023-04-29 09:45:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +26: [2023-04-29 09:45:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +62: [2023-04-29 09:45:05,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +26: [2023-04-29 09:45:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:05,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +62: [2023-04-29 09:45:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +62: [2023-04-29 09:45:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +62: [2023-04-29 09:45:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +13: [2023-04-29 09:45:05,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +13: [2023-04-29 09:45:05,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +13: [2023-04-29 09:45:05,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +28: [2023-04-29 09:45:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +28: [2023-04-29 09:45:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:05,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +13: [2023-04-29 09:45:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +62: [2023-04-29 09:45:05,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +20: [2023-04-29 09:45:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +62: [2023-04-29 09:45:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +23: [2023-04-29 09:45:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +41: [2023-04-29 09:45:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +20: [2023-04-29 09:45:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +45: [2023-04-29 09:45:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +22: [2023-04-29 09:45:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +22: [2023-04-29 09:45:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +22: [2023-04-29 09:45:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +45: [2023-04-29 09:45:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +24: [2023-04-29 09:45:05,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +24: [2023-04-29 09:45:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +24: [2023-04-29 09:45:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +24: [2023-04-29 09:45:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +41: [2023-04-29 09:45:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +21: [2023-04-29 09:45:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +41: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +41: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +11: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +19: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +19: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +19: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +11: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 8: [2023-04-29 09:45:05,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +45: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +45: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 8: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +34: [2023-04-29 09:45:05,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +10: [2023-04-29 09:45:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +10: [2023-04-29 09:45:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +10: [2023-04-29 09:45:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +34: [2023-04-29 09:45:05,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +21: [2023-04-29 09:45:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +21: [2023-04-29 09:45:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +41: [2023-04-29 09:45:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 9: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +21: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +25: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +37: [2023-04-29 09:45:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +37: [2023-04-29 09:45:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +37: [2023-04-29 09:45:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +21: [2023-04-29 09:45:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +37: [2023-04-29 09:45:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +13: [2023-04-29 09:45:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +37: [2023-04-29 09:45:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +37: [2023-04-29 09:45:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +37: [2023-04-29 09:45:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +26: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 7: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +41: [2023-04-29 09:45:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +41: [2023-04-29 09:45:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +13: [2023-04-29 09:45:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +20: [2023-04-29 09:45:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +45: [2023-04-29 09:45:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +50: [2023-04-29 09:45:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +50: [2023-04-29 09:45:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +50: [2023-04-29 09:45:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +20: [2023-04-29 09:45:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +45: [2023-04-29 09:45:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +45: [2023-04-29 09:45:05,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +22: [2023-04-29 09:45:05,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +59: [2023-04-29 09:45:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +60: [2023-04-29 09:45:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +59: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +20: [2023-04-29 09:45:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +55: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +24: [2023-04-29 09:45:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +60: [2023-04-29 09:45:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +60: [2023-04-29 09:45:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +59: [2023-04-29 09:45:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +24: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +47: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +47: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +30: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +30: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +30: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +47: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +19: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +44: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +24: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +59: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +19: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +60: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +20: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +53: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +20: [2023-04-29 09:45:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +47: [2023-04-29 09:45:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +44: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +10: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +43: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +55: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +55: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +59: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +47: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +59: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +47: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +55: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +55: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +55: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +47: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +59: [2023-04-29 09:45:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +59: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +60: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +60: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +60: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +10: [2023-04-29 09:45:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +60: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +10: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +16: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +16: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +16: [2023-04-29 09:45:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +16: [2023-04-29 09:45:05,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +44: [2023-04-29 09:45:05,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +43: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +44: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +44: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +44: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +44: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +44: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +21: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +58: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +58: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_24-model_01-model_states.pt. +53: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +14: [2023-04-29 09:45:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +53: [2023-04-29 09:45:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +53: [2023-04-29 09:45:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +53: [2023-04-29 09:45:05,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:05,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +50: [2023-04-29 09:45:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +43: [2023-04-29 09:45:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +43: [2023-04-29 09:45:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +62: [2023-04-29 09:45:05,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +53: [2023-04-29 09:45:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +50: [2023-04-29 09:45:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +36: [2023-04-29 09:45:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +45: [2023-04-29 09:45:05,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +36: [2023-04-29 09:45:05,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +36: [2023-04-29 09:45:05,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +46: [2023-04-29 09:45:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +36: [2023-04-29 09:45:05,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +46: [2023-04-29 09:45:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +45: [2023-04-29 09:45:05,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +30: [2023-04-29 09:45:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +46: [2023-04-29 09:45:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +47: [2023-04-29 09:45:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +28: [2023-04-29 09:45:05,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +46: [2023-04-29 09:45:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +46: [2023-04-29 09:45:05,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +57: [2023-04-29 09:45:05,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +46: [2023-04-29 09:45:05,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +47: [2023-04-29 09:45:05,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +30: [2023-04-29 09:45:05,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:05,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +47: [2023-04-29 09:45:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +62: [2023-04-29 09:45:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +57: [2023-04-29 09:45:05,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +30: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +57: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +16: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +16: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +14: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +14: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +16: [2023-04-29 09:45:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +62: [2023-04-29 09:45:05,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +28: [2023-04-29 09:45:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +62: [2023-04-29 09:45:05,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +28: [2023-04-29 09:45:05,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +21: [2023-04-29 09:45:05,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +21: [2023-04-29 09:45:05,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +40: [2023-04-29 09:45:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:05,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:05,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +40: [2023-04-29 09:45:05,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +28: [2023-04-29 09:45:05,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +21: [2023-04-29 09:45:05,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +48: [2023-04-29 09:45:05,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +35: [2023-04-29 09:45:05,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +15: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +15: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +33: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +15: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +35: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +15: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +48: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +48: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +28: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +50: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +10: [2023-04-29 09:45:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +10: [2023-04-29 09:45:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +33: [2023-04-29 09:45:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +62: [2023-04-29 09:45:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +33: [2023-04-29 09:45:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +48: [2023-04-29 09:45:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +50: [2023-04-29 09:45:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +33: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +33: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +28: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +28: [2023-04-29 09:45:05,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +10: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +10: [2023-04-29 09:45:05,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +37: [2023-04-29 09:45:05,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +35: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +23: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +35: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +23: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +14: [2023-04-29 09:45:05,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +35: [2023-04-29 09:45:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +35: [2023-04-29 09:45:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +50: [2023-04-29 09:45:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +33: [2023-04-29 09:45:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +33: [2023-04-29 09:45:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +33: [2023-04-29 09:45:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +48: [2023-04-29 09:45:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +33: [2023-04-29 09:45:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +10: [2023-04-29 09:45:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +48: [2023-04-29 09:45:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +48: [2023-04-29 09:45:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +14: [2023-04-29 09:45:05,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +10: [2023-04-29 09:45:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +62: [2023-04-29 09:45:05,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +14: [2023-04-29 09:45:05,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +14: [2023-04-29 09:45:05,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_01-model_01-model_states.pt. +23: [2023-04-29 09:45:05,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +23: [2023-04-29 09:45:05,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +10: [2023-04-29 09:45:05,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +23: [2023-04-29 09:45:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +14: [2023-04-29 09:45:05,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +62: [2023-04-29 09:45:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +14: [2023-04-29 09:45:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +14: [2023-04-29 09:45:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +59: [2023-04-29 09:45:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +11: [2023-04-29 09:45:05,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +37: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +15: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:05,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:05,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +47: [2023-04-29 09:45:05,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +59: [2023-04-29 09:45:05,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +25: [2023-04-29 09:45:05,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +53: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +11: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +44: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +14: [2023-04-29 09:45:05,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +11: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +11: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +55: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +55: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +11: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +59: [2023-04-29 09:45:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +11: [2023-04-29 09:45:05,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +11: [2023-04-29 09:45:05,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +11: [2023-04-29 09:45:05,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +53: [2023-04-29 09:45:05,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +10: [2023-04-29 09:45:05,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +28: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +25: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +44: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +43: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +25: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +25: [2023-04-29 09:45:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +25: [2023-04-29 09:45:05,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +37: [2023-04-29 09:45:05,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +37: [2023-04-29 09:45:05,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +25: [2023-04-29 09:45:05,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:05,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +10: [2023-04-29 09:45:05,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +36: [2023-04-29 09:45:05,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +36: [2023-04-29 09:45:05,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +14: [2023-04-29 09:45:05,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +46: [2023-04-29 09:45:05,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +14: [2023-04-29 09:45:05,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:05,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +50: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +46: [2023-04-29 09:45:05,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +26: [2023-04-29 09:45:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +10: [2023-04-29 09:45:05,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +10: [2023-04-29 09:45:05,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +47: [2023-04-29 09:45:05,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +26: [2023-04-29 09:45:05,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +59: [2023-04-29 09:45:05,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +26: [2023-04-29 09:45:05,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +60: [2023-04-29 09:45:05,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +26: [2023-04-29 09:45:05,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +47: [2023-04-29 09:45:05,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +59: [2023-04-29 09:45:05,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +43: [2023-04-29 09:45:05,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +43: [2023-04-29 09:45:05,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +57: [2023-04-29 09:45:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +60: [2023-04-29 09:45:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +44: [2023-04-29 09:45:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +44: [2023-04-29 09:45:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +53: [2023-04-29 09:45:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +53: [2023-04-29 09:45:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +53: [2023-04-29 09:45:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 1: [2023-04-29 09:45:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +43: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +16: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +43: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +16: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +44: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +16: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +36: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +44: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +53: [2023-04-29 09:45:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +55: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +50: [2023-04-29 09:45:05,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +50: [2023-04-29 09:45:05,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +40: [2023-04-29 09:45:05,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +40: [2023-04-29 09:45:05,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +16: [2023-04-29 09:45:05,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +16: [2023-04-29 09:45:05,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +16: [2023-04-29 09:45:05,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +35: [2023-04-29 09:45:05,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +48: [2023-04-29 09:45:05,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +49: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +36: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +33: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +46: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +16: [2023-04-29 09:45:05,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +36: [2023-04-29 09:45:05,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +16: [2023-04-29 09:45:05,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +16: [2023-04-29 09:45:05,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +16: [2023-04-29 09:45:05,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +49: [2023-04-29 09:45:05,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +35: [2023-04-29 09:45:05,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +16: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +50: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +47: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +47: [2023-04-29 09:45:05,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +35: [2023-04-29 09:45:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +35: [2023-04-29 09:45:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +33: [2023-04-29 09:45:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +48: [2023-04-29 09:45:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +49: [2023-04-29 09:45:05,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +47: [2023-04-29 09:45:05,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +47: [2023-04-29 09:45:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +47: [2023-04-29 09:45:05,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +47: [2023-04-29 09:45:05,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +23: [2023-04-29 09:45:05,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +57: [2023-04-29 09:45:05,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +23: [2023-04-29 09:45:05,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +10: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +10: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +35: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +10: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +48: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 6: [2023-04-29 09:45:05,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +33: [2023-04-29 09:45:05,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +33: [2023-04-29 09:45:05,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +40: [2023-04-29 09:45:05,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +25: [2023-04-29 09:45:05,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +40: [2023-04-29 09:45:05,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +33: [2023-04-29 09:45:05,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +17: [2023-04-29 09:45:05,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +50: [2023-04-29 09:45:05,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +50: [2023-04-29 09:45:05,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +14: [2023-04-29 09:45:05,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +33: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +48: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +35: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +56: [2023-04-29 09:45:05,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:05,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +11: [2023-04-29 09:45:05,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +14: [2023-04-29 09:45:05,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +17: [2023-04-29 09:45:05,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +17: [2023-04-29 09:45:05,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +10: [2023-04-29 09:45:05,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:05,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +23: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +56: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +56: [2023-04-29 09:45:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +23: [2023-04-29 09:45:05,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +25: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +29: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +56: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +29: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +42: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +42: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +23: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +56: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +56: [2023-04-29 09:45:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +56: [2023-04-29 09:45:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +56: [2023-04-29 09:45:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +56: [2023-04-29 09:45:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +23: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +42: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +42: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +42: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +42: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +42: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +14: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +22: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +22: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +56: [2023-04-29 09:45:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +29: [2023-04-29 09:45:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +22: [2023-04-29 09:45:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +56: [2023-04-29 09:45:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +22: [2023-04-29 09:45:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +22: [2023-04-29 09:45:05,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +29: [2023-04-29 09:45:05,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +29: [2023-04-29 09:45:05,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +22: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +25: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +42: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +42: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +29: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +14: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +14: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +42: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +42: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +14: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +42: [2023-04-29 09:45:05,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +25: [2023-04-29 09:45:05,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +42: [2023-04-29 09:45:05,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +50: [2023-04-29 09:45:05,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +42: [2023-04-29 09:45:05,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +50: [2023-04-29 09:45:05,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +14: [2023-04-29 09:45:05,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +58: [2023-04-29 09:45:05,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +16: [2023-04-29 09:45:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +49: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +11: [2023-04-29 09:45:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +58: [2023-04-29 09:45:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:05,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +58: [2023-04-29 09:45:05,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +58: [2023-04-29 09:45:05,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... +26: [2023-04-29 09:45:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +58: [2023-04-29 09:45:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +11: [2023-04-29 09:45:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +58: [2023-04-29 09:45:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... +58: [2023-04-29 09:45:05,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +58: [2023-04-29 09:45:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +25: [2023-04-29 09:45:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +16: [2023-04-29 09:45:05,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +16: [2023-04-29 09:45:05,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +26: [2023-04-29 09:45:05,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +25: [2023-04-29 09:45:05,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +49: [2023-04-29 09:45:05,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +49: [2023-04-29 09:45:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +27: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 8: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +26: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +31: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 8: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 8: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +15: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +15: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +31: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +15: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +31: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +31: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +15: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +16: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 1: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +15: [2023-04-29 09:45:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +12: [2023-04-29 09:45:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +12: [2023-04-29 09:45:05,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +27: [2023-04-29 09:45:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +18: [2023-04-29 09:45:05,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +49: [2023-04-29 09:45:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +27: [2023-04-29 09:45:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +27: [2023-04-29 09:45:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +27: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +19: [2023-04-29 09:45:05,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +18: [2023-04-29 09:45:05,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +18: [2023-04-29 09:45:05,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +13: [2023-04-29 09:45:05,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +18: [2023-04-29 09:45:05,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +18: [2023-04-29 09:45:05,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +13: [2023-04-29 09:45:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +13: [2023-04-29 09:45:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +19: [2023-04-29 09:45:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +13: [2023-04-29 09:45:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +13: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 0: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 0: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 0: [2023-04-29 09:45:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 0: [2023-04-29 09:45:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 0: [2023-04-29 09:45:05,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +19: [2023-04-29 09:45:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +19: [2023-04-29 09:45:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +49: [2023-04-29 09:45:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +19: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +16: [2023-04-29 09:45:05,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +49: [2023-04-29 09:45:05,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +22: [2023-04-29 09:45:05,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +17: [2023-04-29 09:45:05,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 6: [2023-04-29 09:45:05,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +56: [2023-04-29 09:45:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +49: [2023-04-29 09:45:05,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +22: [2023-04-29 09:45:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +17: [2023-04-29 09:45:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +56: [2023-04-29 09:45:05,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +56: [2023-04-29 09:45:05,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 9: [2023-04-29 09:45:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +56: [2023-04-29 09:45:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +42: [2023-04-29 09:45:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +42: [2023-04-29 09:45:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +42: [2023-04-29 09:45:05,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +29: [2023-04-29 09:45:05,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +42: [2023-04-29 09:45:05,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +29: [2023-04-29 09:45:05,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +58: [2023-04-29 09:45:05,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +58: [2023-04-29 09:45:05,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_00-model_states.pt. +22: [2023-04-29 09:45:05,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +56: [2023-04-29 09:45:05,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +17: [2023-04-29 09:45:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +56: [2023-04-29 09:45:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +17: [2023-04-29 09:45:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +42: [2023-04-29 09:45:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +12: [2023-04-29 09:45:05,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +42: [2023-04-29 09:45:05,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +15: [2023-04-29 09:45:05,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +31: [2023-04-29 09:45:05,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +22: [2023-04-29 09:45:05,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +18: [2023-04-29 09:45:05,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +29: [2023-04-29 09:45:05,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +63: [2023-04-29 09:45:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +13: [2023-04-29 09:45:05,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +42: [2023-04-29 09:45:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +18: [2023-04-29 09:45:05,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +42: [2023-04-29 09:45:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +18: [2023-04-29 09:45:05,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +58: [2023-04-29 09:45:05,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +13: [2023-04-29 09:45:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +13: [2023-04-29 09:45:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +63: [2023-04-29 09:45:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +63: [2023-04-29 09:45:05,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:05,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +30: [2023-04-29 09:45:05,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +15: [2023-04-29 09:45:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +15: [2023-04-29 09:45:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +30: [2023-04-29 09:45:05,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:05,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:05,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +30: [2023-04-29 09:45:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... +31: [2023-04-29 09:45:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... +30: [2023-04-29 09:45:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +12: [2023-04-29 09:45:05,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +19: [2023-04-29 09:45:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +27: [2023-04-29 09:45:05,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +15: [2023-04-29 09:45:05,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:05,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +15: [2023-04-29 09:45:05,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +32: [2023-04-29 09:45:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +32: [2023-04-29 09:45:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +32: [2023-04-29 09:45:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +32: [2023-04-29 09:45:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +19: [2023-04-29 09:45:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +63: [2023-04-29 09:45:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +30: [2023-04-29 09:45:05,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +32: [2023-04-29 09:45:05,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +30: [2023-04-29 09:45:05,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +30: [2023-04-29 09:45:05,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_00-model_states.pt. +63: [2023-04-29 09:45:05,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:05,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:05,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +20: [2023-04-29 09:45:05,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +38: [2023-04-29 09:45:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +34: [2023-04-29 09:45:05,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +30: [2023-04-29 09:45:05,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:05,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +52: [2023-04-29 09:45:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +20: [2023-04-29 09:45:05,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +34: [2023-04-29 09:45:05,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +34: [2023-04-29 09:45:05,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +34: [2023-04-29 09:45:05,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +34: [2023-04-29 09:45:05,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +20: [2023-04-29 09:45:05,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:05,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:05,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:05,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +54: [2023-04-29 09:45:05,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +20: [2023-04-29 09:45:05,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +54: [2023-04-29 09:45:05,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +20: [2023-04-29 09:45:05,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +20: [2023-04-29 09:45:05,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:05,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +24: [2023-04-29 09:45:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +24: [2023-04-29 09:45:05,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:05,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:05,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +39: [2023-04-29 09:45:05,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 4: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +62: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +32: [2023-04-29 09:45:05,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +62: [2023-04-29 09:45:05,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +21: [2023-04-29 09:45:05,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +38: [2023-04-29 09:45:05,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +38: [2023-04-29 09:45:05,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +41: [2023-04-29 09:45:05,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +38: [2023-04-29 09:45:05,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +32: [2023-04-29 09:45:05,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +38: [2023-04-29 09:45:05,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +32: [2023-04-29 09:45:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +21: [2023-04-29 09:45:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +21: [2023-04-29 09:45:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +21: [2023-04-29 09:45:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +51: [2023-04-29 09:45:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +32: [2023-04-29 09:45:05,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:05,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:05,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:05,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:05,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +52: [2023-04-29 09:45:05,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:05,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:05,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:05,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:05,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:05,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:05,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +43: [2023-04-29 09:45:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +43: [2023-04-29 09:45:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +43: [2023-04-29 09:45:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +41: [2023-04-29 09:45:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +23: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +23: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +23: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +23: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +47: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +28: [2023-04-29 09:45:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +28: [2023-04-29 09:45:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +28: [2023-04-29 09:45:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +28: [2023-04-29 09:45:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +47: [2023-04-29 09:45:05,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +52: [2023-04-29 09:45:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:05,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:05,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +54: [2023-04-29 09:45:05,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:05,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +39: [2023-04-29 09:45:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +39: [2023-04-29 09:45:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +39: [2023-04-29 09:45:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +62: [2023-04-29 09:45:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +43: [2023-04-29 09:45:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +44: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +44: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +51: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +51: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +39: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +23: [2023-04-29 09:45:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +44: [2023-04-29 09:45:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +46: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +46: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +23: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +44: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +21: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +35: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +53: [2023-04-29 09:45:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +53: [2023-04-29 09:45:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +53: [2023-04-29 09:45:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +51: [2023-04-29 09:45:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +51: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +53: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +62: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +46: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +40: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +45: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +40: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +39: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +63: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +63: [2023-04-29 09:45:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +63: [2023-04-29 09:45:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +35: [2023-04-29 09:45:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +35: [2023-04-29 09:45:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +45: [2023-04-29 09:45:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +35: [2023-04-29 09:45:05,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +52: [2023-04-29 09:45:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +52: [2023-04-29 09:45:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +52: [2023-04-29 09:45:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +28: [2023-04-29 09:45:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +35: [2023-04-29 09:45:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +21: [2023-04-29 09:45:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +41: [2023-04-29 09:45:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +41: [2023-04-29 09:45:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +59: [2023-04-29 09:45:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +48: [2023-04-29 09:45:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +59: [2023-04-29 09:45:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +59: [2023-04-29 09:45:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +48: [2023-04-29 09:45:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +57: [2023-04-29 09:45:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +57: [2023-04-29 09:45:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +48: [2023-04-29 09:45:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +59: [2023-04-29 09:45:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +50: [2023-04-29 09:45:05,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +50: [2023-04-29 09:45:05,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +62: [2023-04-29 09:45:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:05,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +40: [2023-04-29 09:45:05,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +50: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +40: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +45: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +50: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +57: [2023-04-29 09:45:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +57: [2023-04-29 09:45:05,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +57: [2023-04-29 09:45:05,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +10: [2023-04-29 09:45:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:05,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +54: [2023-04-29 09:45:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +37: [2023-04-29 09:45:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:05,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +57: [2023-04-29 09:45:05,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +54: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +44: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +44: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +47: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +47: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +47: [2023-04-29 09:45:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +44: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +46: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +53: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +53: [2023-04-29 09:45:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +53: [2023-04-29 09:45:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +46: [2023-04-29 09:45:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +44: [2023-04-29 09:45:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +53: [2023-04-29 09:45:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +28: [2023-04-29 09:45:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +63: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +60: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +62: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +63: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +28: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +28: [2023-04-29 09:45:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +52: [2023-04-29 09:45:05,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +52: [2023-04-29 09:45:05,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +35: [2023-04-29 09:45:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +63: [2023-04-29 09:45:05,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +35: [2023-04-29 09:45:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +28: [2023-04-29 09:45:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +45: [2023-04-29 09:45:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +28: [2023-04-29 09:45:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +39: [2023-04-29 09:45:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +52: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +21: [2023-04-29 09:45:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +41: [2023-04-29 09:45:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +28: [2023-04-29 09:45:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +59: [2023-04-29 09:45:05,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +59: [2023-04-29 09:45:05,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +41: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +62: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +62: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +45: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +48: [2023-04-29 09:45:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +40: [2023-04-29 09:45:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +57: [2023-04-29 09:45:05,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +41: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +38: [2023-04-29 09:45:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +38: [2023-04-29 09:45:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +56: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +56: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +33: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +33: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +56: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +38: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +38: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +33: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +16: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +16: [2023-04-29 09:45:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +47: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +33: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +47: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +34: [2023-04-29 09:45:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +47: [2023-04-29 09:45:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +36: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +55: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +55: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +55: [2023-04-29 09:45:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +38: [2023-04-29 09:45:05,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +55: [2023-04-29 09:45:05,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +38: [2023-04-29 09:45:05,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 4: [2023-04-29 09:45:05,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +54: [2023-04-29 09:45:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +54: [2023-04-29 09:45:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +34: [2023-04-29 09:45:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +34: [2023-04-29 09:45:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +16: [2023-04-29 09:45:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +16: [2023-04-29 09:45:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +54: [2023-04-29 09:45:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +60: [2023-04-29 09:45:05,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +36: [2023-04-29 09:45:05,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +54: [2023-04-29 09:45:05,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +14: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +61: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +61: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +14: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +14: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +14: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +49: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +49: [2023-04-29 09:45:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +61: [2023-04-29 09:45:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +61: [2023-04-29 09:45:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +35: [2023-04-29 09:45:05,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +24: [2023-04-29 09:45:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +24: [2023-04-29 09:45:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +24: [2023-04-29 09:45:05,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +24: [2023-04-29 09:45:05,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +37: [2023-04-29 09:45:05,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +37: [2023-04-29 09:45:05,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +47: [2023-04-29 09:45:05,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +14: [2023-04-29 09:45:05,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +37: [2023-04-29 09:45:05,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +45: [2023-04-29 09:45:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +59: [2023-04-29 09:45:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +14: [2023-04-29 09:45:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +14: [2023-04-29 09:45:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +14: [2023-04-29 09:45:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:05,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +21: [2023-04-29 09:45:05,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +21: [2023-04-29 09:45:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +21: [2023-04-29 09:45:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +21: [2023-04-29 09:45:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +40: [2023-04-29 09:45:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +59: [2023-04-29 09:45:05,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +58: [2023-04-29 09:45:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +58: [2023-04-29 09:45:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +42: [2023-04-29 09:45:05,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +42: [2023-04-29 09:45:05,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +33: [2023-04-29 09:45:05,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +62: [2023-04-29 09:45:05,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +16: [2023-04-29 09:45:05,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +34: [2023-04-29 09:45:05,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +58: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +58: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +56: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +16: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +58: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +25: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +56: [2023-04-29 09:45:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +42: [2023-04-29 09:45:05,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +36: [2023-04-29 09:45:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +42: [2023-04-29 09:45:05,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_25-model_01-model_states.pt. +47: [2023-04-29 09:45:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +16: [2023-04-29 09:45:05,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +25: [2023-04-29 09:45:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +36: [2023-04-29 09:45:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +40: [2023-04-29 09:45:05,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +34: [2023-04-29 09:45:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +26: [2023-04-29 09:45:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +33: [2023-04-29 09:45:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +39: [2023-04-29 09:45:05,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +25: [2023-04-29 09:45:05,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +26: [2023-04-29 09:45:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +49: [2023-04-29 09:45:05,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +33: [2023-04-29 09:45:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +54: [2023-04-29 09:45:05,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +54: [2023-04-29 09:45:05,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +24: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +61: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +24: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +28: [2023-04-29 09:45:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +36: [2023-04-29 09:45:05,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +26: [2023-04-29 09:45:05,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +26: [2023-04-29 09:45:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +26: [2023-04-29 09:45:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +26: [2023-04-29 09:45:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +54: [2023-04-29 09:45:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +39: [2023-04-29 09:45:05,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +35: [2023-04-29 09:45:05,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +24: [2023-04-29 09:45:05,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +37: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +11: [2023-04-29 09:45:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +11: [2023-04-29 09:45:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 4: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +37: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +37: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +52: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +11: [2023-04-29 09:45:05,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +51: [2023-04-29 09:45:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +11: [2023-04-29 09:45:05,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +57: [2023-04-29 09:45:05,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +28: [2023-04-29 09:45:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +28: [2023-04-29 09:45:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +21: [2023-04-29 09:45:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +40: [2023-04-29 09:45:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +28: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +28: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +57: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +28: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +39: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +52: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +21: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +39: [2023-04-29 09:45:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +21: [2023-04-29 09:45:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +59: [2023-04-29 09:45:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +42: [2023-04-29 09:45:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +63: [2023-04-29 09:45:05,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +63: [2023-04-29 09:45:05,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +39: [2023-04-29 09:45:05,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +63: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +42: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +42: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +28: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +28: [2023-04-29 09:45:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +28: [2023-04-29 09:45:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +39: [2023-04-29 09:45:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +39: [2023-04-29 09:45:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +63: [2023-04-29 09:45:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +51: [2023-04-29 09:45:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +51: [2023-04-29 09:45:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +41: [2023-04-29 09:45:05,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +63: [2023-04-29 09:45:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +52: [2023-04-29 09:45:05,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +10: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +40: [2023-04-29 09:45:05,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +10: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +60: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:05,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +60: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +10: [2023-04-29 09:45:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +41: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +10: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +49: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +41: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +40: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +40: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +52: [2023-04-29 09:45:05,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +48: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +52: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +41: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +52: [2023-04-29 09:45:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +35: [2023-04-29 09:45:05,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +45: [2023-04-29 09:45:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +45: [2023-04-29 09:45:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:05,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +51: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +35: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +11: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +28: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +40: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +40: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +11: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +40: [2023-04-29 09:45:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +11: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +48: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +46: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +57: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +46: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +59: [2023-04-29 09:45:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +48: [2023-04-29 09:45:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +45: [2023-04-29 09:45:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +57: [2023-04-29 09:45:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +10: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +48: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +10: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +41: [2023-04-29 09:45:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 1: [2023-04-29 09:45:05,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +40: [2023-04-29 09:45:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +48: [2023-04-29 09:45:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +14: [2023-04-29 09:45:05,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +28: [2023-04-29 09:45:05,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +28: [2023-04-29 09:45:05,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +56: [2023-04-29 09:45:05,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +56: [2023-04-29 09:45:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +46: [2023-04-29 09:45:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +46: [2023-04-29 09:45:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +46: [2023-04-29 09:45:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +57: [2023-04-29 09:45:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +46: [2023-04-29 09:45:05,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +47: [2023-04-29 09:45:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +57: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +59: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +48: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +48: [2023-04-29 09:45:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +40: [2023-04-29 09:45:05,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +14: [2023-04-29 09:45:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +14: [2023-04-29 09:45:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +14: [2023-04-29 09:45:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +56: [2023-04-29 09:45:05,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +57: [2023-04-29 09:45:05,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +47: [2023-04-29 09:45:05,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +47: [2023-04-29 09:45:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +60: [2023-04-29 09:45:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +10: [2023-04-29 09:45:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +36: [2023-04-29 09:45:05,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +36: [2023-04-29 09:45:05,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +36: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +60: [2023-04-29 09:45:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +47: [2023-04-29 09:45:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +47: [2023-04-29 09:45:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +47: [2023-04-29 09:45:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +25: [2023-04-29 09:45:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +34: [2023-04-29 09:45:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +10: [2023-04-29 09:45:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +60: [2023-04-29 09:45:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +25: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +62: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +35: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +20: [2023-04-29 09:45:05,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +42: [2023-04-29 09:45:05,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +59: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +44: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +10: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +60: [2023-04-29 09:45:05,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +14: [2023-04-29 09:45:05,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +34: [2023-04-29 09:45:05,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +34: [2023-04-29 09:45:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +62: [2023-04-29 09:45:05,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +34: [2023-04-29 09:45:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +44: [2023-04-29 09:45:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +26: [2023-04-29 09:45:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +44: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +35: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +62: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +61: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +25: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +42: [2023-04-29 09:45:05,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 1: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +36: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +43: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +24: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +34: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +43: [2023-04-29 09:45:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +28: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +43: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +60: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +62: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +59: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +26: [2023-04-29 09:45:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +44: [2023-04-29 09:45:05,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +24: [2023-04-29 09:45:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +44: [2023-04-29 09:45:05,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +44: [2023-04-29 09:45:05,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +62: [2023-04-29 09:45:05,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +61: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +61: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +35: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +61: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +61: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +54: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +24: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +43: [2023-04-29 09:45:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +37: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +37: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +36: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +44: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +35: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +44: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +43: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +43: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +21: [2023-04-29 09:45:05,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +49: [2023-04-29 09:45:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +54: [2023-04-29 09:45:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +24: [2023-04-29 09:45:05,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +49: [2023-04-29 09:45:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +59: [2023-04-29 09:45:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +21: [2023-04-29 09:45:05,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +43: [2023-04-29 09:45:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +43: [2023-04-29 09:45:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +61: [2023-04-29 09:45:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +61: [2023-04-29 09:45:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +37: [2023-04-29 09:45:05,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +59: [2023-04-29 09:45:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +21: [2023-04-29 09:45:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +14: [2023-04-29 09:45:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +58: [2023-04-29 09:45:05,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +14: [2023-04-29 09:45:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +43: [2023-04-29 09:45:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +14: [2023-04-29 09:45:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +37: [2023-04-29 09:45:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +37: [2023-04-29 09:45:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +43: [2023-04-29 09:45:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +43: [2023-04-29 09:45:05,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +37: [2023-04-29 09:45:05,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +16: [2023-04-29 09:45:05,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +36: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +21: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +49: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +49: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +21: [2023-04-29 09:45:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:05,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +36: [2023-04-29 09:45:05,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +37: [2023-04-29 09:45:05,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +54: [2023-04-29 09:45:05,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +54: [2023-04-29 09:45:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +54: [2023-04-29 09:45:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +49: [2023-04-29 09:45:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +25: [2023-04-29 09:45:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +37: [2023-04-29 09:45:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +20: [2023-04-29 09:45:05,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 7: [2023-04-29 09:45:05,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 7: [2023-04-29 09:45:05,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 7: [2023-04-29 09:45:05,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +42: [2023-04-29 09:45:05,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:05,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +16: [2023-04-29 09:45:05,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +49: [2023-04-29 09:45:05,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +25: [2023-04-29 09:45:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +21: [2023-04-29 09:45:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +24: [2023-04-29 09:45:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +36: [2023-04-29 09:45:05,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +25: [2023-04-29 09:45:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +25: [2023-04-29 09:45:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +21: [2023-04-29 09:45:05,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +25: [2023-04-29 09:45:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +26: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +53: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +20: [2023-04-29 09:45:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 3: [2023-04-29 09:45:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 3: [2023-04-29 09:45:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 3: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +16: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +16: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +58: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +50: [2023-04-29 09:45:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +20: [2023-04-29 09:45:05,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +58: [2023-04-29 09:45:05,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +58: [2023-04-29 09:45:05,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +58: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +29: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +29: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +29: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +42: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +42: [2023-04-29 09:45:05,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +50: [2023-04-29 09:45:05,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +50: [2023-04-29 09:45:05,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +29: [2023-04-29 09:45:05,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +25: [2023-04-29 09:45:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +50: [2023-04-29 09:45:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +17: [2023-04-29 09:45:05,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +17: [2023-04-29 09:45:05,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +53: [2023-04-29 09:45:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +26: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 2: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 2: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 2: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +42: [2023-04-29 09:45:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +53: [2023-04-29 09:45:05,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +48: [2023-04-29 09:45:05,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +17: [2023-04-29 09:45:05,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +17: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +58: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +53: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +56: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +58: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +48: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +50: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +46: [2023-04-29 09:45:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +58: [2023-04-29 09:45:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +50: [2023-04-29 09:45:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +58: [2023-04-29 09:45:05,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +50: [2023-04-29 09:45:05,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +58: [2023-04-29 09:45:05,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:05,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +56: [2023-04-29 09:45:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +46: [2023-04-29 09:45:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +49: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +49: [2023-04-29 09:45:05,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +48: [2023-04-29 09:45:05,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +25: [2023-04-29 09:45:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +56: [2023-04-29 09:45:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +10: [2023-04-29 09:45:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +33: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +49: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +33: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +33: [2023-04-29 09:45:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +11: [2023-04-29 09:45:05,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +33: [2023-04-29 09:45:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +36: [2023-04-29 09:45:05,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +25: [2023-04-29 09:45:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +33: [2023-04-29 09:45:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... +25: [2023-04-29 09:45:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +29: [2023-04-29 09:45:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +36: [2023-04-29 09:45:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +36: [2023-04-29 09:45:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +56: [2023-04-29 09:45:05,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +11: [2023-04-29 09:45:05,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +25: [2023-04-29 09:45:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +48: [2023-04-29 09:45:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +11: [2023-04-29 09:45:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +56: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +42: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +29: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +48: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +11: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +46: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... +11: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +11: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +44: [2023-04-29 09:45:05,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +46: [2023-04-29 09:45:05,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +56: [2023-04-29 09:45:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +27: [2023-04-29 09:45:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +27: [2023-04-29 09:45:05,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +46: [2023-04-29 09:45:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +46: [2023-04-29 09:45:05,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +17: [2023-04-29 09:45:05,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:05,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +42: [2023-04-29 09:45:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +48: [2023-04-29 09:45:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +43: [2023-04-29 09:45:05,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +44: [2023-04-29 09:45:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +12: [2023-04-29 09:45:05,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +42: [2023-04-29 09:45:05,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +10: [2023-04-29 09:45:05,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +43: [2023-04-29 09:45:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +56: [2023-04-29 09:45:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +12: [2023-04-29 09:45:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +12: [2023-04-29 09:45:05,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 1: [2023-04-29 09:45:05,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 1: [2023-04-29 09:45:05,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +19: [2023-04-29 09:45:05,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +36: [2023-04-29 09:45:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +20: [2023-04-29 09:45:05,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +20: [2023-04-29 09:45:05,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:05,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +20: [2023-04-29 09:45:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +36: [2023-04-29 09:45:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +56: [2023-04-29 09:45:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +18: [2023-04-29 09:45:05,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +18: [2023-04-29 09:45:05,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +44: [2023-04-29 09:45:05,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +18: [2023-04-29 09:45:05,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +18: [2023-04-29 09:45:05,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +22: [2023-04-29 09:45:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +20: [2023-04-29 09:45:05,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:05,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:05,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +22: [2023-04-29 09:45:05,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +22: [2023-04-29 09:45:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +50: [2023-04-29 09:45:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +50: [2023-04-29 09:45:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +27: [2023-04-29 09:45:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +43: [2023-04-29 09:45:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:05,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +53: [2023-04-29 09:45:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +53: [2023-04-29 09:45:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:05,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +44: [2023-04-29 09:45:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +44: [2023-04-29 09:45:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +44: [2023-04-29 09:45:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +27: [2023-04-29 09:45:05,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +53: [2023-04-29 09:45:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +42: [2023-04-29 09:45:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +50: [2023-04-29 09:45:05,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +43: [2023-04-29 09:45:05,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:05,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +12: [2023-04-29 09:45:05,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +50: [2023-04-29 09:45:05,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +13: [2023-04-29 09:45:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +19: [2023-04-29 09:45:05,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +13: [2023-04-29 09:45:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +25: [2023-04-29 09:45:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +55: [2023-04-29 09:45:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +55: [2023-04-29 09:45:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +33: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +15: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +15: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +15: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +15: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +22: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +22: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +25: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +20: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 0: [2023-04-29 09:45:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +18: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +22: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +22: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +50: [2023-04-29 09:45:05,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +22: [2023-04-29 09:45:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +24: [2023-04-29 09:45:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +16: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +53: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +17: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +18: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +22: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +50: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +24: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +20: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +53: [2023-04-29 09:45:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +24: [2023-04-29 09:45:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:05,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +22: [2023-04-29 09:45:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +53: [2023-04-29 09:45:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:05,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_00-model_states.pt. +17: [2023-04-29 09:45:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +16: [2023-04-29 09:45:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +11: [2023-04-29 09:45:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +25: [2023-04-29 09:45:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +50: [2023-04-29 09:45:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +11: [2023-04-29 09:45:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 8: [2023-04-29 09:45:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 8: [2023-04-29 09:45:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 8: [2023-04-29 09:45:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 3: [2023-04-29 09:45:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +50: [2023-04-29 09:45:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +13: [2023-04-29 09:45:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +25: [2023-04-29 09:45:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +25: [2023-04-29 09:45:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +29: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +29: [2023-04-29 09:45:05,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +33: [2023-04-29 09:45:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +25: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +17: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +15: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 6: [2023-04-29 09:45:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 5: [2023-04-29 09:45:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 6: [2023-04-29 09:45:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 5: [2023-04-29 09:45:05,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +15: [2023-04-29 09:45:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +29: [2023-04-29 09:45:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +55: [2023-04-29 09:45:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +29: [2023-04-29 09:45:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +55: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +14: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +14: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +55: [2023-04-29 09:45:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +29: [2023-04-29 09:45:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +29: [2023-04-29 09:45:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +14: [2023-04-29 09:45:05,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 1: [2023-04-29 09:45:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +15: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +29: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +15: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 1: [2023-04-29 09:45:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:05,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +33: [2023-04-29 09:45:05,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +19: [2023-04-29 09:45:05,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +31: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +11: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +20: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +23: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 6: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +12: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +17: [2023-04-29 09:45:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +23: [2023-04-29 09:45:05,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +12: [2023-04-29 09:45:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +11: [2023-04-29 09:45:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 8: [2023-04-29 09:45:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +19: [2023-04-29 09:45:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +21: [2023-04-29 09:45:05,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:05,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +20: [2023-04-29 09:45:05,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +20: [2023-04-29 09:45:05,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:05,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:05,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +26: [2023-04-29 09:45:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +26: [2023-04-29 09:45:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +26: [2023-04-29 09:45:05,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 5: [2023-04-29 09:45:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:05,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:05,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:05,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:05,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:05,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:05,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:05,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +14: [2023-04-29 09:45:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 6: [2023-04-29 09:45:05,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:05,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:05,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +20: [2023-04-29 09:45:05,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:05,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +19: [2023-04-29 09:45:05,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +12: [2023-04-29 09:45:05,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +22: [2023-04-29 09:45:05,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +12: [2023-04-29 09:45:05,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +14: [2023-04-29 09:45:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 7: [2023-04-29 09:45:05,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 7: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 2: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +27: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +17: [2023-04-29 09:45:05,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 9: [2023-04-29 09:45:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +17: [2023-04-29 09:45:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 9: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +27: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +30: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. + 9: [2023-04-29 09:45:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +27: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +24: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 3: [2023-04-29 09:45:05,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +27: [2023-04-29 09:45:05,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +26: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +22: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +26: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +17: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:05,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +24: [2023-04-29 09:45:05,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:05,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +20: [2023-04-29 09:45:05,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:05,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +26: [2023-04-29 09:45:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +18: [2023-04-29 09:45:05,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +18: [2023-04-29 09:45:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +19: [2023-04-29 09:45:05,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +29: [2023-04-29 09:45:05,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:05,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +31: [2023-04-29 09:45:05,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +31: [2023-04-29 09:45:05,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +18: [2023-04-29 09:45:05,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +18: [2023-04-29 09:45:05,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +20: [2023-04-29 09:45:05,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:05,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 2: [2023-04-29 09:45:05,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:05,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +31: [2023-04-29 09:45:05,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_03-model_01-model_states.pt. +19: [2023-04-29 09:45:05,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 4: [2023-04-29 09:45:05,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:05,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +22: [2023-04-29 09:45:05,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:05,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:05,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:05,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:05,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:05,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:05,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:05,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +23: [2023-04-29 09:45:05,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:05,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:05,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:05,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:05,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:05,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:05,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:05,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:05,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +17: [2023-04-29 09:45:05,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:05,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:05,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:05,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:05,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +12: [2023-04-29 09:45:06,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:06,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +23: [2023-04-29 09:45:06,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +29: [2023-04-29 09:45:06,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:06,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +24: [2023-04-29 09:45:06,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:06,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +10: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +14: [2023-04-29 09:45:06,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:06,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +10: [2023-04-29 09:45:06,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +63: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +31: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +31: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +10: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +63: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +10: [2023-04-29 09:45:06,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +10: [2023-04-29 09:45:06,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:06,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +14: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +30: [2023-04-29 09:45:06,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +13: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +13: [2023-04-29 09:45:06,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +13: [2023-04-29 09:45:06,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +13: [2023-04-29 09:45:06,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:06,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +30: [2023-04-29 09:45:06,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:06,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +19: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +63: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:06,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:06,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:06,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +23: [2023-04-29 09:45:06,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:06,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:06,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +51: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +51: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +51: [2023-04-29 09:45:06,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +21: [2023-04-29 09:45:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:06,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:06,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +34: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +34: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +34: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +39: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +39: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +51: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +23: [2023-04-29 09:45:06,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +12: [2023-04-29 09:45:06,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:06,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +34: [2023-04-29 09:45:06,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +34: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +15: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +39: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +39: [2023-04-29 09:45:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +39: [2023-04-29 09:45:06,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:06,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:06,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +39: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +27: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +32: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +21: [2023-04-29 09:45:06,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +18: [2023-04-29 09:45:06,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +15: [2023-04-29 09:45:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +31: [2023-04-29 09:45:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:06,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +32: [2023-04-29 09:45:06,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:06,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +31: [2023-04-29 09:45:06,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +32: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +15: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +15: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +27: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +32: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +15: [2023-04-29 09:45:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +26: [2023-04-29 09:45:06,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt... +52: [2023-04-29 09:45:06,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +41: [2023-04-29 09:45:06,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +41: [2023-04-29 09:45:06,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +41: [2023-04-29 09:45:06,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +41: [2023-04-29 09:45:06,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +18: [2023-04-29 09:45:06,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +52: [2023-04-29 09:45:06,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +52: [2023-04-29 09:45:06,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +52: [2023-04-29 09:45:06,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +52: [2023-04-29 09:45:06,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +21: [2023-04-29 09:45:06,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +41: [2023-04-29 09:45:06,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +41: [2023-04-29 09:45:06,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +41: [2023-04-29 09:45:06,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:06,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:06,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:06,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +14: [2023-04-29 09:45:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +26: [2023-04-29 09:45:06,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +14: [2023-04-29 09:45:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +19: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +54: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +21: [2023-04-29 09:45:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +19: [2023-04-29 09:45:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +19: [2023-04-29 09:45:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +19: [2023-04-29 09:45:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +18: [2023-04-29 09:45:06,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:06,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:06,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +47: [2023-04-29 09:45:06,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +14: [2023-04-29 09:45:06,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +47: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +54: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +38: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +54: [2023-04-29 09:45:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +27: [2023-04-29 09:45:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +47: [2023-04-29 09:45:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:06,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +30: [2023-04-29 09:45:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +27: [2023-04-29 09:45:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +18: [2023-04-29 09:45:06,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +38: [2023-04-29 09:45:06,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +30: [2023-04-29 09:45:06,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +51: [2023-04-29 09:45:06,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +10: [2023-04-29 09:45:06,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +18: [2023-04-29 09:45:06,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... + 9: [2023-04-29 09:45:06,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +63: [2023-04-29 09:45:06,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +45: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +34: [2023-04-29 09:45:06,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +13: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +51: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +13: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +13: [2023-04-29 09:45:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +63: [2023-04-29 09:45:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +26: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +26: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +26: [2023-04-29 09:45:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +45: [2023-04-29 09:45:06,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +45: [2023-04-29 09:45:06,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +30: [2023-04-29 09:45:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +30: [2023-04-29 09:45:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +45: [2023-04-29 09:45:06,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +45: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +39: [2023-04-29 09:45:06,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +26: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +62: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +31: [2023-04-29 09:45:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +26: [2023-04-29 09:45:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +34: [2023-04-29 09:45:06,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +26: [2023-04-29 09:45:06,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:06,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +39: [2023-04-29 09:45:06,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +39: [2023-04-29 09:45:06,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +51: [2023-04-29 09:45:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +62: [2023-04-29 09:45:06,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +30: [2023-04-29 09:45:06,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +32: [2023-04-29 09:45:06,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +31: [2023-04-29 09:45:06,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +62: [2023-04-29 09:45:06,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:06,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +30: [2023-04-29 09:45:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +31: [2023-04-29 09:45:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt... +52: [2023-04-29 09:45:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +30: [2023-04-29 09:45:06,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +51: [2023-04-29 09:45:06,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +32: [2023-04-29 09:45:06,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +32: [2023-04-29 09:45:06,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +32: [2023-04-29 09:45:06,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +41: [2023-04-29 09:45:06,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +28: [2023-04-29 09:45:06,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +51: [2023-04-29 09:45:06,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +34: [2023-04-29 09:45:06,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +34: [2023-04-29 09:45:06,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +15: [2023-04-29 09:45:06,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +32: [2023-04-29 09:45:06,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +10: [2023-04-29 09:45:06,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:06,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +15: [2023-04-29 09:45:06,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +34: [2023-04-29 09:45:06,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +15: [2023-04-29 09:45:06,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_00-model_states.pt. +63: [2023-04-29 09:45:06,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:06,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +13: [2023-04-29 09:45:06,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +63: [2023-04-29 09:45:06,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +52: [2023-04-29 09:45:06,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +41: [2023-04-29 09:45:06,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +41: [2023-04-29 09:45:06,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +39: [2023-04-29 09:45:06,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:06,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +63: [2023-04-29 09:45:06,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +34: [2023-04-29 09:45:06,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +51: [2023-04-29 09:45:06,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +10: [2023-04-29 09:45:06,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:06,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +52: [2023-04-29 09:45:06,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +41: [2023-04-29 09:45:06,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +41: [2023-04-29 09:45:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +30: [2023-04-29 09:45:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +28: [2023-04-29 09:45:06,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +54: [2023-04-29 09:45:06,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +14: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +32: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +32: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +28: [2023-04-29 09:45:06,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:06,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +32: [2023-04-29 09:45:06,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:06,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +34: [2023-04-29 09:45:06,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +32: [2023-04-29 09:45:06,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +15: [2023-04-29 09:45:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +47: [2023-04-29 09:45:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +32: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +15: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +32: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +32: [2023-04-29 09:45:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +52: [2023-04-29 09:45:06,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:06,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +41: [2023-04-29 09:45:06,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +41: [2023-04-29 09:45:06,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +45: [2023-04-29 09:45:06,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:06,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +37: [2023-04-29 09:45:06,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:06,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +38: [2023-04-29 09:45:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +41: [2023-04-29 09:45:06,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +26: [2023-04-29 09:45:06,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +62: [2023-04-29 09:45:06,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +26: [2023-04-29 09:45:06,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +26: [2023-04-29 09:45:06,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +54: [2023-04-29 09:45:06,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:06,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:06,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +60: [2023-04-29 09:45:06,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:06,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:06,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:06,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +40: [2023-04-29 09:45:06,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:06,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +14: [2023-04-29 09:45:06,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +60: [2023-04-29 09:45:06,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +14: [2023-04-29 09:45:06,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +60: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +60: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +62: [2023-04-29 09:45:06,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +62: [2023-04-29 09:45:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +58: [2023-04-29 09:45:06,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:06,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +35: [2023-04-29 09:45:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:06,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +54: [2023-04-29 09:45:06,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +54: [2023-04-29 09:45:06,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +35: [2023-04-29 09:45:06,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:06,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:06,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:06,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +26: [2023-04-29 09:45:06,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +57: [2023-04-29 09:45:06,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +48: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +28: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +28: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +28: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +57: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +46: [2023-04-29 09:45:06,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +46: [2023-04-29 09:45:06,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +46: [2023-04-29 09:45:06,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:06,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +45: [2023-04-29 09:45:06,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +45: [2023-04-29 09:45:06,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:06,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:06,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +36: [2023-04-29 09:45:06,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:06,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:06,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +16: [2023-04-29 09:45:06,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +36: [2023-04-29 09:45:06,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +26: [2023-04-29 09:45:06,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +36: [2023-04-29 09:45:06,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +32: [2023-04-29 09:45:06,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +32: [2023-04-29 09:45:06,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +25: [2023-04-29 09:45:06,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +42: [2023-04-29 09:45:06,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:06,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +32: [2023-04-29 09:45:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +25: [2023-04-29 09:45:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +42: [2023-04-29 09:45:06,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:06,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:06,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:06,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:06,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +56: [2023-04-29 09:45:06,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +56: [2023-04-29 09:45:06,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +56: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +24: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +24: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +44: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +24: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +24: [2023-04-29 09:45:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +44: [2023-04-29 09:45:06,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +44: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:06,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:06,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +28: [2023-04-29 09:45:06,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +58: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +11: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +53: [2023-04-29 09:45:06,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:06,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:06,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +28: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +33: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:06,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +11: [2023-04-29 09:45:06,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +59: [2023-04-29 09:45:06,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +60: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +60: [2023-04-29 09:45:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:06,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:06,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:06,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +50: [2023-04-29 09:45:06,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:06,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:06,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:06,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:06,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:06,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +35: [2023-04-29 09:45:06,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +40: [2023-04-29 09:45:06,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +35: [2023-04-29 09:45:06,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +24: [2023-04-29 09:45:06,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +46: [2023-04-29 09:45:06,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +24: [2023-04-29 09:45:06,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +40: [2023-04-29 09:45:06,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:06,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:06,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:06,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:06,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:06,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:06,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:06,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:06,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +58: [2023-04-29 09:45:06,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +58: [2023-04-29 09:45:06,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:06,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +37: [2023-04-29 09:45:06,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +25: [2023-04-29 09:45:06,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +37: [2023-04-29 09:45:06,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:06,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:06,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:06,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +58: [2023-04-29 09:45:06,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +25: [2023-04-29 09:45:06,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +49: [2023-04-29 09:45:06,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:06,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:06,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:06,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:06,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:06,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:06,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +35: [2023-04-29 09:45:06,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +25: [2023-04-29 09:45:06,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +46: [2023-04-29 09:45:06,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +35: [2023-04-29 09:45:06,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +52: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +52: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +33: [2023-04-29 09:45:06,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:06,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:06,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +53: [2023-04-29 09:45:06,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +16: [2023-04-29 09:45:06,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +57: [2023-04-29 09:45:06,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:06,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +53: [2023-04-29 09:45:06,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:06,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:06,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:06,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +11: [2023-04-29 09:45:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +38: [2023-04-29 09:45:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +38: [2023-04-29 09:45:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +38: [2023-04-29 09:45:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +38: [2023-04-29 09:45:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +25: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +56: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +21: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +62: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +62: [2023-04-29 09:45:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +33: [2023-04-29 09:45:06,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:06,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +53: [2023-04-29 09:45:06,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +33: [2023-04-29 09:45:06,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +21: [2023-04-29 09:45:06,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +44: [2023-04-29 09:45:06,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +21: [2023-04-29 09:45:06,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +42: [2023-04-29 09:45:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:06,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +43: [2023-04-29 09:45:06,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +33: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:06,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:06,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +52: [2023-04-29 09:45:06,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +43: [2023-04-29 09:45:06,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +24: [2023-04-29 09:45:06,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +50: [2023-04-29 09:45:06,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:06,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +52: [2023-04-29 09:45:06,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +53: [2023-04-29 09:45:06,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:06,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +24: [2023-04-29 09:45:06,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 6: [2023-04-29 09:45:06,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +48: [2023-04-29 09:45:06,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +48: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +48: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +48: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +24: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +28: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +28: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +28: [2023-04-29 09:45:06,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +53: [2023-04-29 09:45:06,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +16: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 1: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +11: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 1: [2023-04-29 09:45:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +55: [2023-04-29 09:45:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +24: [2023-04-29 09:45:06,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +50: [2023-04-29 09:45:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +24: [2023-04-29 09:45:06,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +62: [2023-04-29 09:45:06,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +38: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +38: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +44: [2023-04-29 09:45:06,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +62: [2023-04-29 09:45:06,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +62: [2023-04-29 09:45:06,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +33: [2023-04-29 09:45:06,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +33: [2023-04-29 09:45:06,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +53: [2023-04-29 09:45:06,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:06,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:06,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +24: [2023-04-29 09:45:06,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +62: [2023-04-29 09:45:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +11: [2023-04-29 09:45:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +11: [2023-04-29 09:45:06,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +55: [2023-04-29 09:45:06,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:06,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +39: [2023-04-29 09:45:06,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +39: [2023-04-29 09:45:06,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +39: [2023-04-29 09:45:06,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +20: [2023-04-29 09:45:06,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +54: [2023-04-29 09:45:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +54: [2023-04-29 09:45:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +54: [2023-04-29 09:45:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +54: [2023-04-29 09:45:06,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +43: [2023-04-29 09:45:06,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +20: [2023-04-29 09:45:06,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +43: [2023-04-29 09:45:06,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +20: [2023-04-29 09:45:06,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +55: [2023-04-29 09:45:06,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +20: [2023-04-29 09:45:06,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:06,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +48: [2023-04-29 09:45:06,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:06,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +48: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +55: [2023-04-29 09:45:06,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +39: [2023-04-29 09:45:06,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:06,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:06,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:06,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:06,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +21: [2023-04-29 09:45:06,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +21: [2023-04-29 09:45:06,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +21: [2023-04-29 09:45:06,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +54: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +54: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +10: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +10: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +54: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +54: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +52: [2023-04-29 09:45:06,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +56: [2023-04-29 09:45:06,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +56: [2023-04-29 09:45:06,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +56: [2023-04-29 09:45:06,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +52: [2023-04-29 09:45:06,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:06,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +52: [2023-04-29 09:45:06,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +38: [2023-04-29 09:45:06,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +52: [2023-04-29 09:45:06,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +10: [2023-04-29 09:45:06,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +38: [2023-04-29 09:45:06,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +38: [2023-04-29 09:45:06,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +10: [2023-04-29 09:45:06,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +62: [2023-04-29 09:45:06,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +52: [2023-04-29 09:45:06,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +38: [2023-04-29 09:45:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +38: [2023-04-29 09:45:06,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +62: [2023-04-29 09:45:06,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +38: [2023-04-29 09:45:06,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +52: [2023-04-29 09:45:06,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +52: [2023-04-29 09:45:06,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +20: [2023-04-29 09:45:06,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +62: [2023-04-29 09:45:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +10: [2023-04-29 09:45:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +10: [2023-04-29 09:45:06,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +56: [2023-04-29 09:45:06,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +62: [2023-04-29 09:45:06,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +62: [2023-04-29 09:45:06,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +63: [2023-04-29 09:45:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +63: [2023-04-29 09:45:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +63: [2023-04-29 09:45:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +62: [2023-04-29 09:45:06,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +21: [2023-04-29 09:45:06,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +56: [2023-04-29 09:45:06,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:06,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:06,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +10: [2023-04-29 09:45:06,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +56: [2023-04-29 09:45:06,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +56: [2023-04-29 09:45:06,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +62: [2023-04-29 09:45:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +62: [2023-04-29 09:45:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +14: [2023-04-29 09:45:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +24: [2023-04-29 09:45:06,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:06,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +14: [2023-04-29 09:45:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:06,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +21: [2023-04-29 09:45:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +21: [2023-04-29 09:45:06,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +21: [2023-04-29 09:45:06,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +48: [2023-04-29 09:45:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +24: [2023-04-29 09:45:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +20: [2023-04-29 09:45:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +48: [2023-04-29 09:45:06,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +16: [2023-04-29 09:45:06,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +39: [2023-04-29 09:45:06,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +20: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +11: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +63: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +39: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +39: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +48: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +63: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +11: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +39: [2023-04-29 09:45:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +16: [2023-04-29 09:45:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +16: [2023-04-29 09:45:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +39: [2023-04-29 09:45:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +16: [2023-04-29 09:45:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +16: [2023-04-29 09:45:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +54: [2023-04-29 09:45:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +12: [2023-04-29 09:45:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +12: [2023-04-29 09:45:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +22: [2023-04-29 09:45:06,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +54: [2023-04-29 09:45:06,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +54: [2023-04-29 09:45:06,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +22: [2023-04-29 09:45:06,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +12: [2023-04-29 09:45:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +54: [2023-04-29 09:45:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +54: [2023-04-29 09:45:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +54: [2023-04-29 09:45:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +12: [2023-04-29 09:45:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:06,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +20: [2023-04-29 09:45:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +49: [2023-04-29 09:45:06,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +49: [2023-04-29 09:45:06,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +11: [2023-04-29 09:45:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +49: [2023-04-29 09:45:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +49: [2023-04-29 09:45:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +50: [2023-04-29 09:45:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +50: [2023-04-29 09:45:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +28: [2023-04-29 09:45:06,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +50: [2023-04-29 09:45:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +20: [2023-04-29 09:45:06,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +50: [2023-04-29 09:45:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +56: [2023-04-29 09:45:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +10: [2023-04-29 09:45:06,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +28: [2023-04-29 09:45:06,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +59: [2023-04-29 09:45:06,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +59: [2023-04-29 09:45:06,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +59: [2023-04-29 09:45:06,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +59: [2023-04-29 09:45:06,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +56: [2023-04-29 09:45:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +56: [2023-04-29 09:45:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +21: [2023-04-29 09:45:06,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 2: [2023-04-29 09:45:06,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +56: [2023-04-29 09:45:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +56: [2023-04-29 09:45:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +17: [2023-04-29 09:45:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 3: [2023-04-29 09:45:06,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 3: [2023-04-29 09:45:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 3: [2023-04-29 09:45:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +17: [2023-04-29 09:45:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +60: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +60: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +60: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +56: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +56: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:06,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +14: [2023-04-29 09:45:06,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +49: [2023-04-29 09:45:06,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:06,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +17: [2023-04-29 09:45:06,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:06,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +17: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +21: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:06,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:06,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +50: [2023-04-29 09:45:06,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +31: [2023-04-29 09:45:06,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:06,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:06,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +63: [2023-04-29 09:45:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +50: [2023-04-29 09:45:06,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +49: [2023-04-29 09:45:06,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +50: [2023-04-29 09:45:06,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +31: [2023-04-29 09:45:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:06,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +37: [2023-04-29 09:45:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +37: [2023-04-29 09:45:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +37: [2023-04-29 09:45:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +37: [2023-04-29 09:45:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +10: [2023-04-29 09:45:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +30: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +30: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +49: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +21: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +30: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +50: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +31: [2023-04-29 09:45:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +10: [2023-04-29 09:45:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +31: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +31: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +36: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +14: [2023-04-29 09:45:06,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +63: [2023-04-29 09:45:06,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +63: [2023-04-29 09:45:06,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +30: [2023-04-29 09:45:06,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +25: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +30: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +25: [2023-04-29 09:45:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +24: [2023-04-29 09:45:06,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +59: [2023-04-29 09:45:06,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +63: [2023-04-29 09:45:06,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +59: [2023-04-29 09:45:06,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +63: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +24: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +60: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +60: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +63: [2023-04-29 09:45:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +51: [2023-04-29 09:45:06,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +51: [2023-04-29 09:45:06,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +63: [2023-04-29 09:45:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +60: [2023-04-29 09:45:06,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +22: [2023-04-29 09:45:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +26: [2023-04-29 09:45:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +26: [2023-04-29 09:45:06,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +51: [2023-04-29 09:45:06,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 5: [2023-04-29 09:45:06,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +12: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +23: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +23: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +23: [2023-04-29 09:45:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +12: [2023-04-29 09:45:06,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +26: [2023-04-29 09:45:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +22: [2023-04-29 09:45:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:06,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +22: [2023-04-29 09:45:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +26: [2023-04-29 09:45:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +12: [2023-04-29 09:45:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +37: [2023-04-29 09:45:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +37: [2023-04-29 09:45:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +37: [2023-04-29 09:45:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:06,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +10: [2023-04-29 09:45:06,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +36: [2023-04-29 09:45:06,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:06,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +10: [2023-04-29 09:45:06,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +36: [2023-04-29 09:45:06,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:06,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +24: [2023-04-29 09:45:06,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:06,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +16: [2023-04-29 09:45:06,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +11: [2023-04-29 09:45:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +11: [2023-04-29 09:45:06,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 9: [2023-04-29 09:45:06,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +51: [2023-04-29 09:45:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:06,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +51: [2023-04-29 09:45:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +22: [2023-04-29 09:45:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +51: [2023-04-29 09:45:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +23: [2023-04-29 09:45:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +49: [2023-04-29 09:45:06,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +16: [2023-04-29 09:45:06,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +51: [2023-04-29 09:45:06,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +12: [2023-04-29 09:45:06,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +22: [2023-04-29 09:45:06,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +50: [2023-04-29 09:45:06,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +49: [2023-04-29 09:45:06,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +49: [2023-04-29 09:45:06,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +22: [2023-04-29 09:45:06,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 6: [2023-04-29 09:45:06,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +28: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +34: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +34: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +18: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +49: [2023-04-29 09:45:06,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +29: [2023-04-29 09:45:06,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +25: [2023-04-29 09:45:06,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +29: [2023-04-29 09:45:06,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +34: [2023-04-29 09:45:06,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +25: [2023-04-29 09:45:06,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +34: [2023-04-29 09:45:06,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +31: [2023-04-29 09:45:06,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +50: [2023-04-29 09:45:06,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +18: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +20: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +20: [2023-04-29 09:45:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +20: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +20: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +40: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +40: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +18: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +30: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +27: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +49: [2023-04-29 09:45:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +18: [2023-04-29 09:45:06,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:06,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:06,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:06,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +50: [2023-04-29 09:45:06,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +50: [2023-04-29 09:45:06,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +17: [2023-04-29 09:45:06,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:06,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +30: [2023-04-29 09:45:06,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +50: [2023-04-29 09:45:06,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +30: [2023-04-29 09:45:06,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +40: [2023-04-29 09:45:06,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +17: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +40: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +60: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +27: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +60: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +27: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:06,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +49: [2023-04-29 09:45:06,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +50: [2023-04-29 09:45:06,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +13: [2023-04-29 09:45:06,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +13: [2023-04-29 09:45:06,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +13: [2023-04-29 09:45:06,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +13: [2023-04-29 09:45:06,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +50: [2023-04-29 09:45:06,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +35: [2023-04-29 09:45:06,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +35: [2023-04-29 09:45:06,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +60: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +44: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +44: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +44: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +59: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +35: [2023-04-29 09:45:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +35: [2023-04-29 09:45:06,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +13: [2023-04-29 09:45:06,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +28: [2023-04-29 09:45:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +60: [2023-04-29 09:45:06,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +26: [2023-04-29 09:45:06,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +37: [2023-04-29 09:45:06,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +59: [2023-04-29 09:45:06,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +26: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +34: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +23: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +23: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +59: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +34: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:06,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +23: [2023-04-29 09:45:06,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +60: [2023-04-29 09:45:06,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +34: [2023-04-29 09:45:06,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +36: [2023-04-29 09:45:06,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +36: [2023-04-29 09:45:06,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +20: [2023-04-29 09:45:06,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +37: [2023-04-29 09:45:06,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +31: [2023-04-29 09:45:06,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +37: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +26: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +26: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +40: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +23: [2023-04-29 09:45:06,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +34: [2023-04-29 09:45:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +23: [2023-04-29 09:45:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:06,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +23: [2023-04-29 09:45:06,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +36: [2023-04-29 09:45:06,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +36: [2023-04-29 09:45:06,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +42: [2023-04-29 09:45:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +42: [2023-04-29 09:45:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +42: [2023-04-29 09:45:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +37: [2023-04-29 09:45:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +37: [2023-04-29 09:45:06,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +37: [2023-04-29 09:45:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +20: [2023-04-29 09:45:06,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +19: [2023-04-29 09:45:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +19: [2023-04-29 09:45:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +19: [2023-04-29 09:45:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +29: [2023-04-29 09:45:06,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +29: [2023-04-29 09:45:06,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +40: [2023-04-29 09:45:06,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +17: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +47: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +47: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +47: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +47: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +37: [2023-04-29 09:45:06,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +40: [2023-04-29 09:45:06,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +20: [2023-04-29 09:45:06,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +53: [2023-04-29 09:45:06,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +53: [2023-04-29 09:45:06,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +30: [2023-04-29 09:45:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:06,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:06,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +53: [2023-04-29 09:45:06,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +30: [2023-04-29 09:45:06,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +53: [2023-04-29 09:45:06,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +43: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +37: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +51: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +43: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +43: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +35: [2023-04-29 09:45:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +35: [2023-04-29 09:45:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +51: [2023-04-29 09:45:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +44: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +44: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +44: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 1: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 1: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +44: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +17: [2023-04-29 09:45:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +61: [2023-04-29 09:45:06,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +61: [2023-04-29 09:45:06,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +61: [2023-04-29 09:45:06,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +61: [2023-04-29 09:45:06,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +36: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +25: [2023-04-29 09:45:06,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +35: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +17: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +36: [2023-04-29 09:45:06,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +35: [2023-04-29 09:45:06,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +17: [2023-04-29 09:45:06,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +15: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +51: [2023-04-29 09:45:06,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +58: [2023-04-29 09:45:06,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +15: [2023-04-29 09:45:06,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +15: [2023-04-29 09:45:06,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +51: [2023-04-29 09:45:06,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +51: [2023-04-29 09:45:06,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +23: [2023-04-29 09:45:06,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +58: [2023-04-29 09:45:06,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +58: [2023-04-29 09:45:06,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +15: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +11: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +51: [2023-04-29 09:45:06,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +55: [2023-04-29 09:45:06,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +55: [2023-04-29 09:45:06,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +55: [2023-04-29 09:45:06,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +55: [2023-04-29 09:45:06,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +25: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +41: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +41: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 7: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +53: [2023-04-29 09:45:06,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +18: [2023-04-29 09:45:06,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +57: [2023-04-29 09:45:06,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +57: [2023-04-29 09:45:06,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +15: [2023-04-29 09:45:06,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +19: [2023-04-29 09:45:06,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +19: [2023-04-29 09:45:06,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +57: [2023-04-29 09:45:06,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +57: [2023-04-29 09:45:06,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +15: [2023-04-29 09:45:06,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +42: [2023-04-29 09:45:06,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +26: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +42: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +53: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +51: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +47: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +42: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +53: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:06,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +23: [2023-04-29 09:45:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +23: [2023-04-29 09:45:06,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +23: [2023-04-29 09:45:06,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:06,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +26: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +16: [2023-04-29 09:45:06,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +42: [2023-04-29 09:45:06,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +33: [2023-04-29 09:45:06,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +47: [2023-04-29 09:45:06,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:06,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +47: [2023-04-29 09:45:06,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +25: [2023-04-29 09:45:06,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +25: [2023-04-29 09:45:06,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +33: [2023-04-29 09:45:06,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +33: [2023-04-29 09:45:06,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +19: [2023-04-29 09:45:06,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:06,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:06,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +43: [2023-04-29 09:45:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +43: [2023-04-29 09:45:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +43: [2023-04-29 09:45:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 9: [2023-04-29 09:45:06,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +61: [2023-04-29 09:45:06,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +27: [2023-04-29 09:45:06,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +19: [2023-04-29 09:45:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +61: [2023-04-29 09:45:06,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +27: [2023-04-29 09:45:06,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +27: [2023-04-29 09:45:06,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +27: [2023-04-29 09:45:06,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +27: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 9: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +61: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +61: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +27: [2023-04-29 09:45:06,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +58: [2023-04-29 09:45:06,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +58: [2023-04-29 09:45:06,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +34: [2023-04-29 09:45:06,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +18: [2023-04-29 09:45:06,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:06,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +58: [2023-04-29 09:45:06,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +13: [2023-04-29 09:45:06,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +58: [2023-04-29 09:45:06,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:06,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +12: [2023-04-29 09:45:06,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +12: [2023-04-29 09:45:06,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +34: [2023-04-29 09:45:06,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +57: [2023-04-29 09:45:06,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:06,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +57: [2023-04-29 09:45:06,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +55: [2023-04-29 09:45:06,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +16: [2023-04-29 09:45:06,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:06,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +34: [2023-04-29 09:45:06,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +12: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +41: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +12: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +13: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +13: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +13: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +55: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +55: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:06,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:06,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:06,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +40: [2023-04-29 09:45:06,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +23: [2023-04-29 09:45:06,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +20: [2023-04-29 09:45:06,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +34: [2023-04-29 09:45:06,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +34: [2023-04-29 09:45:06,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +16: [2023-04-29 09:45:06,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +33: [2023-04-29 09:45:06,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +41: [2023-04-29 09:45:06,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +40: [2023-04-29 09:45:06,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +40: [2023-04-29 09:45:06,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +33: [2023-04-29 09:45:06,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +18: [2023-04-29 09:45:06,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +40: [2023-04-29 09:45:06,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +20: [2023-04-29 09:45:06,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 6: [2023-04-29 09:45:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:06,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +34: [2023-04-29 09:45:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +34: [2023-04-29 09:45:06,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +40: [2023-04-29 09:45:06,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +27: [2023-04-29 09:45:06,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +44: [2023-04-29 09:45:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +23: [2023-04-29 09:45:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +40: [2023-04-29 09:45:06,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +27: [2023-04-29 09:45:06,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +44: [2023-04-29 09:45:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +20: [2023-04-29 09:45:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +20: [2023-04-29 09:45:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +44: [2023-04-29 09:45:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +32: [2023-04-29 09:45:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +34: [2023-04-29 09:45:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +32: [2023-04-29 09:45:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +32: [2023-04-29 09:45:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +32: [2023-04-29 09:45:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +40: [2023-04-29 09:45:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +23: [2023-04-29 09:45:06,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +27: [2023-04-29 09:45:06,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +27: [2023-04-29 09:45:06,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +44: [2023-04-29 09:45:06,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +44: [2023-04-29 09:45:06,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +35: [2023-04-29 09:45:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +13: [2023-04-29 09:45:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +27: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +20: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +17: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +17: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +17: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +17: [2023-04-29 09:45:06,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +20: [2023-04-29 09:45:06,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +27: [2023-04-29 09:45:06,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +20: [2023-04-29 09:45:06,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +35: [2023-04-29 09:45:06,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +44: [2023-04-29 09:45:06,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 6: [2023-04-29 09:45:06,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +12: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +53: [2023-04-29 09:45:06,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +35: [2023-04-29 09:45:06,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +12: [2023-04-29 09:45:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +25: [2023-04-29 09:45:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +35: [2023-04-29 09:45:06,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +44: [2023-04-29 09:45:06,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +20: [2023-04-29 09:45:06,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +53: [2023-04-29 09:45:06,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +13: [2023-04-29 09:45:06,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:06,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +15: [2023-04-29 09:45:06,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +35: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 8: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 8: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +42: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +35: [2023-04-29 09:45:06,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +53: [2023-04-29 09:45:06,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +53: [2023-04-29 09:45:06,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +23: [2023-04-29 09:45:06,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:06,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:06,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +47: [2023-04-29 09:45:06,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +42: [2023-04-29 09:45:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +53: [2023-04-29 09:45:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +11: [2023-04-29 09:45:06,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +35: [2023-04-29 09:45:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +53: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +15: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +53: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +47: [2023-04-29 09:45:06,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +47: [2023-04-29 09:45:06,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +42: [2023-04-29 09:45:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +42: [2023-04-29 09:45:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +11: [2023-04-29 09:45:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:06,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +61: [2023-04-29 09:45:06,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +32: [2023-04-29 09:45:06,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:06,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +47: [2023-04-29 09:45:06,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +23: [2023-04-29 09:45:06,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +32: [2023-04-29 09:45:06,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +32: [2023-04-29 09:45:06,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:06,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +42: [2023-04-29 09:45:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +32: [2023-04-29 09:45:06,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:06,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +19: [2023-04-29 09:45:06,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +43: [2023-04-29 09:45:06,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +61: [2023-04-29 09:45:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +43: [2023-04-29 09:45:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +43: [2023-04-29 09:45:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +23: [2023-04-29 09:45:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +47: [2023-04-29 09:45:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +61: [2023-04-29 09:45:06,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +58: [2023-04-29 09:45:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +45: [2023-04-29 09:45:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +45: [2023-04-29 09:45:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +45: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +45: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +58: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 2: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +61: [2023-04-29 09:45:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +43: [2023-04-29 09:45:06,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +43: [2023-04-29 09:45:06,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +43: [2023-04-29 09:45:06,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +58: [2023-04-29 09:45:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +61: [2023-04-29 09:45:06,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +15: [2023-04-29 09:45:06,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +58: [2023-04-29 09:45:06,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +15: [2023-04-29 09:45:06,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +38: [2023-04-29 09:45:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +38: [2023-04-29 09:45:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +38: [2023-04-29 09:45:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +38: [2023-04-29 09:45:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 2: [2023-04-29 09:45:06,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +57: [2023-04-29 09:45:06,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +61: [2023-04-29 09:45:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +17: [2023-04-29 09:45:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 2: [2023-04-29 09:45:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +61: [2023-04-29 09:45:06,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +58: [2023-04-29 09:45:06,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +58: [2023-04-29 09:45:06,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +33: [2023-04-29 09:45:06,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +52: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +52: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +52: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +52: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +57: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +57: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +57: [2023-04-29 09:45:06,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +58: [2023-04-29 09:45:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +55: [2023-04-29 09:45:06,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +58: [2023-04-29 09:45:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +11: [2023-04-29 09:45:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +11: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +41: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +17: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +41: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +17: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +33: [2023-04-29 09:45:06,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +11: [2023-04-29 09:45:06,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +57: [2023-04-29 09:45:06,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +15: [2023-04-29 09:45:06,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +41: [2023-04-29 09:45:06,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +41: [2023-04-29 09:45:06,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +57: [2023-04-29 09:45:06,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +55: [2023-04-29 09:45:06,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +41: [2023-04-29 09:45:06,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 1: [2023-04-29 09:45:06,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +41: [2023-04-29 09:45:06,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +55: [2023-04-29 09:45:06,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 7: [2023-04-29 09:45:06,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +33: [2023-04-29 09:45:06,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +27: [2023-04-29 09:45:06,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +55: [2023-04-29 09:45:06,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +19: [2023-04-29 09:45:06,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +41: [2023-04-29 09:45:06,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +57: [2023-04-29 09:45:06,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +41: [2023-04-29 09:45:06,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +19: [2023-04-29 09:45:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +33: [2023-04-29 09:45:06,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +33: [2023-04-29 09:45:06,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +55: [2023-04-29 09:45:06,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +45: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +45: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +33: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +45: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +45: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +28: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +28: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +27: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +22: [2023-04-29 09:45:06,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +22: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +22: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +13: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +13: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +22: [2023-04-29 09:45:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +55: [2023-04-29 09:45:06,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:06,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +13: [2023-04-29 09:45:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +26: [2023-04-29 09:45:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +26: [2023-04-29 09:45:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +26: [2023-04-29 09:45:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +38: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +26: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +38: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +33: [2023-04-29 09:45:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +33: [2023-04-29 09:45:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +38: [2023-04-29 09:45:06,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:06,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +19: [2023-04-29 09:45:06,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +12: [2023-04-29 09:45:06,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 1: [2023-04-29 09:45:06,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:06,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +52: [2023-04-29 09:45:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +19: [2023-04-29 09:45:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +52: [2023-04-29 09:45:06,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +27: [2023-04-29 09:45:06,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +12: [2023-04-29 09:45:06,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +12: [2023-04-29 09:45:06,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +52: [2023-04-29 09:45:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +27: [2023-04-29 09:45:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +46: [2023-04-29 09:45:06,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +46: [2023-04-29 09:45:06,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +12: [2023-04-29 09:45:06,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +12: [2023-04-29 09:45:06,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +46: [2023-04-29 09:45:06,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +46: [2023-04-29 09:45:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_26-model_01-model_states.pt. +12: [2023-04-29 09:45:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +27: [2023-04-29 09:45:06,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 6: [2023-04-29 09:45:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +27: [2023-04-29 09:45:06,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +12: [2023-04-29 09:45:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 7: [2023-04-29 09:45:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:06,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +13: [2023-04-29 09:45:06,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +26: [2023-04-29 09:45:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +28: [2023-04-29 09:45:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +46: [2023-04-29 09:45:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +46: [2023-04-29 09:45:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +46: [2023-04-29 09:45:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... +28: [2023-04-29 09:45:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +46: [2023-04-29 09:45:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 6: [2023-04-29 09:45:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +17: [2023-04-29 09:45:06,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +45: [2023-04-29 09:45:06,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +17: [2023-04-29 09:45:06,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 8: [2023-04-29 09:45:06,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 9: [2023-04-29 09:45:06,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 9: [2023-04-29 09:45:06,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 9: [2023-04-29 09:45:06,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +45: [2023-04-29 09:45:06,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 9: [2023-04-29 09:45:06,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +17: [2023-04-29 09:45:06,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +17: [2023-04-29 09:45:06,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +45: [2023-04-29 09:45:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +45: [2023-04-29 09:45:06,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +45: [2023-04-29 09:45:06,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +17: [2023-04-29 09:45:06,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +17: [2023-04-29 09:45:06,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 2: [2023-04-29 09:45:06,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +45: [2023-04-29 09:45:06,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +45: [2023-04-29 09:45:06,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +24: [2023-04-29 09:45:06,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +24: [2023-04-29 09:45:06,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +24: [2023-04-29 09:45:06,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +24: [2023-04-29 09:45:06,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +45: [2023-04-29 09:45:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 0: [2023-04-29 09:45:06,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +21: [2023-04-29 09:45:06,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +21: [2023-04-29 09:45:06,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +21: [2023-04-29 09:45:06,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +21: [2023-04-29 09:45:06,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 0: [2023-04-29 09:45:06,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:06,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:06,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:06,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:06,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:06,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:06,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +30: [2023-04-29 09:45:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +46: [2023-04-29 09:45:06,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +22: [2023-04-29 09:45:06,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +22: [2023-04-29 09:45:06,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:06,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +13: [2023-04-29 09:45:06,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +13: [2023-04-29 09:45:06,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +13: [2023-04-29 09:45:06,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 2: [2023-04-29 09:45:06,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 2: [2023-04-29 09:45:06,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +46: [2023-04-29 09:45:06,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +46: [2023-04-29 09:45:06,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:06,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +26: [2023-04-29 09:45:06,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 2: [2023-04-29 09:45:06,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +26: [2023-04-29 09:45:06,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:06,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +13: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +46: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_00-model_states.pt. +46: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... + 2: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +24: [2023-04-29 09:45:06,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +22: [2023-04-29 09:45:06,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +46: [2023-04-29 09:45:06,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +46: [2023-04-29 09:45:06,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt... +26: [2023-04-29 09:45:06,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +24: [2023-04-29 09:45:06,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +26: [2023-04-29 09:45:06,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +29: [2023-04-29 09:45:06,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +22: [2023-04-29 09:45:06,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:06,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:06,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +28: [2023-04-29 09:45:06,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +29: [2023-04-29 09:45:06,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:06,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:06,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +29: [2023-04-29 09:45:06,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +28: [2023-04-29 09:45:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +10: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +10: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +10: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +10: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +11: [2023-04-29 09:45:06,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +11: [2023-04-29 09:45:06,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +11: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +11: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +30: [2023-04-29 09:45:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:06,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:06,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:06,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:06,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +62: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +62: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +14: [2023-04-29 09:45:06,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +62: [2023-04-29 09:45:06,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +14: [2023-04-29 09:45:06,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +62: [2023-04-29 09:45:06,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +14: [2023-04-29 09:45:06,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:06,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:06,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:06,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +54: [2023-04-29 09:45:06,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +54: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +21: [2023-04-29 09:45:06,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +21: [2023-04-29 09:45:06,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +24: [2023-04-29 09:45:06,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:06,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:06,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:06,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +22: [2023-04-29 09:45:06,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +10: [2023-04-29 09:45:06,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +39: [2023-04-29 09:45:06,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +39: [2023-04-29 09:45:06,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 9: [2023-04-29 09:45:06,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:06,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:06,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +31: [2023-04-29 09:45:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +31: [2023-04-29 09:45:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +39: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +31: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +31: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +22: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +24: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:06,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:06,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +31: [2023-04-29 09:45:06,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:06,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:06,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:06,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 5: [2023-04-29 09:45:06,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +29: [2023-04-29 09:45:06,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +62: [2023-04-29 09:45:06,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +30: [2023-04-29 09:45:06,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:06,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +10: [2023-04-29 09:45:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +10: [2023-04-29 09:45:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +10: [2023-04-29 09:45:06,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +20: [2023-04-29 09:45:06,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +20: [2023-04-29 09:45:06,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:06,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:06,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +62: [2023-04-29 09:45:06,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +62: [2023-04-29 09:45:06,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +30: [2023-04-29 09:45:06,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +62: [2023-04-29 09:45:06,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +17: [2023-04-29 09:45:06,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +21: [2023-04-29 09:45:06,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +24: [2023-04-29 09:45:06,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:06,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +24: [2023-04-29 09:45:06,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +21: [2023-04-29 09:45:06,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +10: [2023-04-29 09:45:06,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:06,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +10: [2023-04-29 09:45:06,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +54: [2023-04-29 09:45:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +10: [2023-04-29 09:45:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +22: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +29: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +39: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:06,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:06,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +21: [2023-04-29 09:45:06,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +39: [2023-04-29 09:45:06,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +20: [2023-04-29 09:45:06,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +28: [2023-04-29 09:45:06,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:06,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +24: [2023-04-29 09:45:06,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +28: [2023-04-29 09:45:06,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:06,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:06,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 5: [2023-04-29 09:45:06,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:06,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +28: [2023-04-29 09:45:06,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:06,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:06,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +14: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +15: [2023-04-29 09:45:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +15: [2023-04-29 09:45:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +15: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +14: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +14: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 5: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +14: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +29: [2023-04-29 09:45:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +29: [2023-04-29 09:45:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +34: [2023-04-29 09:45:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +34: [2023-04-29 09:45:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +34: [2023-04-29 09:45:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +29: [2023-04-29 09:45:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +34: [2023-04-29 09:45:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:06,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:06,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +34: [2023-04-29 09:45:06,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +18: [2023-04-29 09:45:06,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +18: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +25: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +18: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +34: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +12: [2023-04-29 09:45:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +34: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +34: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 5: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 5: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 9: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 9: [2023-04-29 09:45:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:06,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +18: [2023-04-29 09:45:06,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:06,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:06,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:06,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:06,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +18: [2023-04-29 09:45:06,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +21: [2023-04-29 09:45:06,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:06,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +24: [2023-04-29 09:45:06,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:06,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +45: [2023-04-29 09:45:06,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:06,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:06,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:06,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +31: [2023-04-29 09:45:06,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:06,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:06,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:06,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +21: [2023-04-29 09:45:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:06,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +45: [2023-04-29 09:45:06,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +23: [2023-04-29 09:45:06,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +45: [2023-04-29 09:45:06,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +45: [2023-04-29 09:45:06,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +15: [2023-04-29 09:45:06,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +45: [2023-04-29 09:45:06,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +31: [2023-04-29 09:45:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 4: [2023-04-29 09:45:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:06,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +26: [2023-04-29 09:45:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:06,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:06,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +26: [2023-04-29 09:45:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:06,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +29: [2023-04-29 09:45:06,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +29: [2023-04-29 09:45:06,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +16: [2023-04-29 09:45:06,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +16: [2023-04-29 09:45:06,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +29: [2023-04-29 09:45:06,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +16: [2023-04-29 09:45:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +14: [2023-04-29 09:45:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:06,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:06,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +49: [2023-04-29 09:45:06,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +49: [2023-04-29 09:45:06,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +29: [2023-04-29 09:45:06,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +24: [2023-04-29 09:45:06,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:06,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +31: [2023-04-29 09:45:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +49: [2023-04-29 09:45:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +49: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 6: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 6: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 6: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +56: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +56: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 6: [2023-04-29 09:45:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +56: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +56: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +12: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +48: [2023-04-29 09:45:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +39: [2023-04-29 09:45:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +39: [2023-04-29 09:45:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +10: [2023-04-29 09:45:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:06,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:06,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 5: [2023-04-29 09:45:06,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +48: [2023-04-29 09:45:06,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +48: [2023-04-29 09:45:06,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +10: [2023-04-29 09:45:06,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +21: [2023-04-29 09:45:06,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +11: [2023-04-29 09:45:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +39: [2023-04-29 09:45:06,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +39: [2023-04-29 09:45:06,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:06,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +39: [2023-04-29 09:45:06,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +24: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +25: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +10: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +50: [2023-04-29 09:45:06,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +25: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +16: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +31: [2023-04-29 09:45:06,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:06,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:06,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +25: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +27: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +27: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +39: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +16: [2023-04-29 09:45:06,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +10: [2023-04-29 09:45:06,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:06,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +31: [2023-04-29 09:45:06,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +21: [2023-04-29 09:45:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:06,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +39: [2023-04-29 09:45:06,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +21: [2023-04-29 09:45:06,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:06,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:06,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +27: [2023-04-29 09:45:06,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:06,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +16: [2023-04-29 09:45:06,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:06,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:06,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +39: [2023-04-29 09:45:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +39: [2023-04-29 09:45:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +16: [2023-04-29 09:45:06,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:06,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +49: [2023-04-29 09:45:06,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:06,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +28: [2023-04-29 09:45:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +16: [2023-04-29 09:45:06,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +48: [2023-04-29 09:45:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +39: [2023-04-29 09:45:06,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +10: [2023-04-29 09:45:06,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +31: [2023-04-29 09:45:06,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +16: [2023-04-29 09:45:06,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +15: [2023-04-29 09:45:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +39: [2023-04-29 09:45:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +24: [2023-04-29 09:45:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +56: [2023-04-29 09:45:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +48: [2023-04-29 09:45:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +16: [2023-04-29 09:45:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +16: [2023-04-29 09:45:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:06,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:06,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:06,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +56: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +24: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +10: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +49: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +10: [2023-04-29 09:45:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +49: [2023-04-29 09:45:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 6: [2023-04-29 09:45:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +28: [2023-04-29 09:45:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +39: [2023-04-29 09:45:06,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +28: [2023-04-29 09:45:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +34: [2023-04-29 09:45:06,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:06,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +18: [2023-04-29 09:45:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:06,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 9: [2023-04-29 09:45:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:06,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +50: [2023-04-29 09:45:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +17: [2023-04-29 09:45:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:06,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:06,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:06,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:06,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:06,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:06,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:06,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:06,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:06,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +13: [2023-04-29 09:45:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +34: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +34: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +34: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +50: [2023-04-29 09:45:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:06,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +50: [2023-04-29 09:45:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +18: [2023-04-29 09:45:06,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:06,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +13: [2023-04-29 09:45:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:06,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +25: [2023-04-29 09:45:06,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:06,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:06,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:06,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:06,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:06,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:06,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 3: [2023-04-29 09:45:06,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 3: [2023-04-29 09:45:06,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 4: [2023-04-29 09:45:06,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:06,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:06,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:06,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:06,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +45: [2023-04-29 09:45:07,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +23: [2023-04-29 09:45:07,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +48: [2023-04-29 09:45:07,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:07,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:07,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:07,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +26: [2023-04-29 09:45:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +15: [2023-04-29 09:45:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +48: [2023-04-29 09:45:07,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +29: [2023-04-29 09:45:07,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:07,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +28: [2023-04-29 09:45:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +15: [2023-04-29 09:45:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +10: [2023-04-29 09:45:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +23: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +45: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +15: [2023-04-29 09:45:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +25: [2023-04-29 09:45:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +54: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +37: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +37: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +15: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +37: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +15: [2023-04-29 09:45:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +31: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +31: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +54: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +20: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +29: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +36: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +36: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +36: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +36: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +60: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +60: [2023-04-29 09:45:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +60: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +31: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +60: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +23: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +63: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +63: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +63: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +31: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +54: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +54: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +54: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +54: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:07,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +25: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +34: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +34: [2023-04-29 09:45:07,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +25: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +39: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +39: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +18: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +18: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +18: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +34: [2023-04-29 09:45:07,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +23: [2023-04-29 09:45:07,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +29: [2023-04-29 09:45:07,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +18: [2023-04-29 09:45:07,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 4: [2023-04-29 09:45:07,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +39: [2023-04-29 09:45:07,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +29: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +29: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +54: [2023-04-29 09:45:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +18: [2023-04-29 09:45:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +20: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +54: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +54: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +25: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +54: [2023-04-29 09:45:07,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:07,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +54: [2023-04-29 09:45:07,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +25: [2023-04-29 09:45:07,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:07,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:07,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +54: [2023-04-29 09:45:07,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +45: [2023-04-29 09:45:07,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +20: [2023-04-29 09:45:07,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:07,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +26: [2023-04-29 09:45:07,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:07,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +19: [2023-04-29 09:45:07,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +19: [2023-04-29 09:45:07,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +19: [2023-04-29 09:45:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +19: [2023-04-29 09:45:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 8: [2023-04-29 09:45:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +27: [2023-04-29 09:45:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +14: [2023-04-29 09:45:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +14: [2023-04-29 09:45:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +16: [2023-04-29 09:45:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:07,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +45: [2023-04-29 09:45:07,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +48: [2023-04-29 09:45:07,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +45: [2023-04-29 09:45:07,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +27: [2023-04-29 09:45:07,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +37: [2023-04-29 09:45:07,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +16: [2023-04-29 09:45:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +16: [2023-04-29 09:45:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +37: [2023-04-29 09:45:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +60: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +25: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +36: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +31: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +37: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +13: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +35: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +35: [2023-04-29 09:45:07,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +14: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +58: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +58: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +31: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +14: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +63: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +27: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +63: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +58: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +63: [2023-04-29 09:45:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +31: [2023-04-29 09:45:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +31: [2023-04-29 09:45:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +39: [2023-04-29 09:45:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +37: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +37: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +15: [2023-04-29 09:45:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +63: [2023-04-29 09:45:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +58: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +16: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +58: [2023-04-29 09:45:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +58: [2023-04-29 09:45:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +48: [2023-04-29 09:45:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:07,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +15: [2023-04-29 09:45:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:07,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:07,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +16: [2023-04-29 09:45:07,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:07,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +18: [2023-04-29 09:45:07,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +37: [2023-04-29 09:45:07,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +15: [2023-04-29 09:45:07,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +37: [2023-04-29 09:45:07,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:07,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +37: [2023-04-29 09:45:07,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +37: [2023-04-29 09:45:07,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +25: [2023-04-29 09:45:07,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +34: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +16: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +16: [2023-04-29 09:45:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +34: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +16: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +19: [2023-04-29 09:45:07,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:07,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +16: [2023-04-29 09:45:07,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:07,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:07,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:07,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +11: [2023-04-29 09:45:07,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:07,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:07,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +23: [2023-04-29 09:45:07,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +19: [2023-04-29 09:45:07,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:07,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +27: [2023-04-29 09:45:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +27: [2023-04-29 09:45:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +13: [2023-04-29 09:45:07,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +40: [2023-04-29 09:45:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +40: [2023-04-29 09:45:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +12: [2023-04-29 09:45:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +35: [2023-04-29 09:45:07,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +16: [2023-04-29 09:45:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +40: [2023-04-29 09:45:07,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +16: [2023-04-29 09:45:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +35: [2023-04-29 09:45:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:07,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +27: [2023-04-29 09:45:07,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +27: [2023-04-29 09:45:07,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +36: [2023-04-29 09:45:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +36: [2023-04-29 09:45:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +15: [2023-04-29 09:45:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +15: [2023-04-29 09:45:07,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +35: [2023-04-29 09:45:07,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +20: [2023-04-29 09:45:07,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +36: [2023-04-29 09:45:07,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +19: [2023-04-29 09:45:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +36: [2023-04-29 09:45:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +48: [2023-04-29 09:45:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +20: [2023-04-29 09:45:07,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +34: [2023-04-29 09:45:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +34: [2023-04-29 09:45:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +16: [2023-04-29 09:45:07,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:07,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +34: [2023-04-29 09:45:07,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +48: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +48: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +23: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +59: [2023-04-29 09:45:07,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +59: [2023-04-29 09:45:07,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +16: [2023-04-29 09:45:07,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +59: [2023-04-29 09:45:07,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +48: [2023-04-29 09:45:07,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +59: [2023-04-29 09:45:07,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +12: [2023-04-29 09:45:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +23: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +23: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +48: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +53: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +53: [2023-04-29 09:45:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +13: [2023-04-29 09:45:07,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +40: [2023-04-29 09:45:07,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:07,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +27: [2023-04-29 09:45:07,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +27: [2023-04-29 09:45:07,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +20: [2023-04-29 09:45:07,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +20: [2023-04-29 09:45:07,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:07,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +40: [2023-04-29 09:45:07,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +19: [2023-04-29 09:45:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +53: [2023-04-29 09:45:07,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:07,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +53: [2023-04-29 09:45:07,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +54: [2023-04-29 09:45:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +53: [2023-04-29 09:45:07,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +54: [2023-04-29 09:45:07,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +54: [2023-04-29 09:45:07,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +20: [2023-04-29 09:45:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +20: [2023-04-29 09:45:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +30: [2023-04-29 09:45:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +30: [2023-04-29 09:45:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +31: [2023-04-29 09:45:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +54: [2023-04-29 09:45:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +30: [2023-04-29 09:45:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +30: [2023-04-29 09:45:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_04-model_01-model_states.pt. +27: [2023-04-29 09:45:07,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +27: [2023-04-29 09:45:07,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:07,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:07,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +31: [2023-04-29 09:45:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +31: [2023-04-29 09:45:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +58: [2023-04-29 09:45:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +58: [2023-04-29 09:45:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +31: [2023-04-29 09:45:07,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +51: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +51: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +37: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +61: [2023-04-29 09:45:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +18: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +31: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +51: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +31: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +38: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +58: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +58: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +63: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +55: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +51: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +47: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +47: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +47: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +37: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +41: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +41: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +37: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +18: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +62: [2023-04-29 09:45:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +51: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +51: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +38: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +38: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +38: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +42: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +38: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +51: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +32: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +32: [2023-04-29 09:45:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +32: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +49: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +38: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +32: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +46: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +63: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +55: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +32: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +44: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +63: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +55: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +55: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +43: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +56: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +55: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +60: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +47: [2023-04-29 09:45:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +56: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +62: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +42: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +42: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +56: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +60: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +63: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +60: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +11: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +63: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +46: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +63: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +51: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +40: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +35: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +47: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +62: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +52: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +52: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +18: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +18: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +18: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +35: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +63: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +52: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 8: [2023-04-29 09:45:07,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +62: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 2: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +40: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +49: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +40: [2023-04-29 09:45:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +56: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +42: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +49: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +49: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +60: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +50: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +44: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +37: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +37: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +37: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +56: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +44: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +52: [2023-04-29 09:45:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +46: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +60: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +60: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +50: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +35: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +35: [2023-04-29 09:45:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +11: [2023-04-29 09:45:07,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +35: [2023-04-29 09:45:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +35: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +30: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +59: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +62: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +62: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +18: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +19: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +62: [2023-04-29 09:45:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +30: [2023-04-29 09:45:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +30: [2023-04-29 09:45:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt... +37: [2023-04-29 09:45:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +49: [2023-04-29 09:45:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:07,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +49: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +49: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +19: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +54: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +54: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +36: [2023-04-29 09:45:07,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +35: [2023-04-29 09:45:07,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +58: [2023-04-29 09:45:07,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +54: [2023-04-29 09:45:07,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:07,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +59: [2023-04-29 09:45:07,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +59: [2023-04-29 09:45:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +19: [2023-04-29 09:45:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +17: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +17: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +17: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +19: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +17: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +58: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +35: [2023-04-29 09:45:07,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +58: [2023-04-29 09:45:07,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +30: [2023-04-29 09:45:07,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +35: [2023-04-29 09:45:07,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +51: [2023-04-29 09:45:07,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +37: [2023-04-29 09:45:07,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +34: [2023-04-29 09:45:07,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +34: [2023-04-29 09:45:07,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +51: [2023-04-29 09:45:07,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +22: [2023-04-29 09:45:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +34: [2023-04-29 09:45:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +30: [2023-04-29 09:45:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +36: [2023-04-29 09:45:07,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +36: [2023-04-29 09:45:07,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +11: [2023-04-29 09:45:07,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +35: [2023-04-29 09:45:07,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +34: [2023-04-29 09:45:07,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +34: [2023-04-29 09:45:07,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:07,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +43: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +11: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +34: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +34: [2023-04-29 09:45:07,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:07,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +35: [2023-04-29 09:45:07,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +37: [2023-04-29 09:45:07,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +42: [2023-04-29 09:45:07,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +42: [2023-04-29 09:45:07,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +12: [2023-04-29 09:45:07,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +11: [2023-04-29 09:45:07,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:07,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +37: [2023-04-29 09:45:07,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +37: [2023-04-29 09:45:07,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +34: [2023-04-29 09:45:07,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +12: [2023-04-29 09:45:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +53: [2023-04-29 09:45:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +12: [2023-04-29 09:45:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:07,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +53: [2023-04-29 09:45:07,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +27: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +53: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +27: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +40: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +40: [2023-04-29 09:45:07,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:07,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +27: [2023-04-29 09:45:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 2: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 2: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:07,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +17: [2023-04-29 09:45:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +40: [2023-04-29 09:45:07,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:07,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +27: [2023-04-29 09:45:07,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +19: [2023-04-29 09:45:07,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +27: [2023-04-29 09:45:07,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +40: [2023-04-29 09:45:07,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +17: [2023-04-29 09:45:07,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +40: [2023-04-29 09:45:07,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +19: [2023-04-29 09:45:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +60: [2023-04-29 09:45:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +22: [2023-04-29 09:45:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +36: [2023-04-29 09:45:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +38: [2023-04-29 09:45:07,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +36: [2023-04-29 09:45:07,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +27: [2023-04-29 09:45:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +41: [2023-04-29 09:45:07,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +22: [2023-04-29 09:45:07,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +22: [2023-04-29 09:45:07,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +63: [2023-04-29 09:45:07,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +32: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +43: [2023-04-29 09:45:07,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +62: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +43: [2023-04-29 09:45:07,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:07,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +41: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +52: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +38: [2023-04-29 09:45:07,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +49: [2023-04-29 09:45:07,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +50: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +32: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +27: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +57: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +63: [2023-04-29 09:45:07,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +32: [2023-04-29 09:45:07,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +47: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +47: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +33: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +30: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +32: [2023-04-29 09:45:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +38: [2023-04-29 09:45:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +40: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +53: [2023-04-29 09:45:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +35: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +30: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +46: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +56: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +42: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +59: [2023-04-29 09:45:07,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +50: [2023-04-29 09:45:07,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +35: [2023-04-29 09:45:07,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +59: [2023-04-29 09:45:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +51: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +62: [2023-04-29 09:45:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +49: [2023-04-29 09:45:07,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +60: [2023-04-29 09:45:07,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +50: [2023-04-29 09:45:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +49: [2023-04-29 09:45:07,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +49: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +62: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +38: [2023-04-29 09:45:07,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +59: [2023-04-29 09:45:07,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +59: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +61: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +12: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +59: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +22: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +40: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +49: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +56: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +22: [2023-04-29 09:45:07,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:07,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +56: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +60: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +22: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +43: [2023-04-29 09:45:07,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +30: [2023-04-29 09:45:07,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. +35: [2023-04-29 09:45:07,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +50: [2023-04-29 09:45:07,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +44: [2023-04-29 09:45:07,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +47: [2023-04-29 09:45:07,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +52: [2023-04-29 09:45:07,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +63: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +57: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +62: [2023-04-29 09:45:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +38: [2023-04-29 09:45:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +38: [2023-04-29 09:45:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +43: [2023-04-29 09:45:07,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +47: [2023-04-29 09:45:07,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +47: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +47: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +56: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +44: [2023-04-29 09:45:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +44: [2023-04-29 09:45:07,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +44: [2023-04-29 09:45:07,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +38: [2023-04-29 09:45:07,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt... +57: [2023-04-29 09:45:07,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +51: [2023-04-29 09:45:07,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +56: [2023-04-29 09:45:07,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +42: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +42: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +55: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +49: [2023-04-29 09:45:07,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +62: [2023-04-29 09:45:07,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +62: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +51: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +58: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +58: [2023-04-29 09:45:07,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +35: [2023-04-29 09:45:07,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +59: [2023-04-29 09:45:07,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +58: [2023-04-29 09:45:07,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 8: [2023-04-29 09:45:07,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +59: [2023-04-29 09:45:07,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 2: [2023-04-29 09:45:07,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +43: [2023-04-29 09:45:07,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +49: [2023-04-29 09:45:07,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +13: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +40: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +29: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +53: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +57: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +17: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +17: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +49: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +53: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +57: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +57: [2023-04-29 09:45:07,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +57: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +13: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +42: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +29: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +29: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +51: [2023-04-29 09:45:07,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +46: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +46: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +41: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +41: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +41: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +51: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +59: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +55: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +55: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +55: [2023-04-29 09:45:07,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +46: [2023-04-29 09:45:07,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +46: [2023-04-29 09:45:07,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +53: [2023-04-29 09:45:07,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +42: [2023-04-29 09:45:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +43: [2023-04-29 09:45:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +43: [2023-04-29 09:45:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +30: [2023-04-29 09:45:07,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +29: [2023-04-29 09:45:07,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +59: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +17: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +29: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +42: [2023-04-29 09:45:07,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +30: [2023-04-29 09:45:07,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +33: [2023-04-29 09:45:07,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +17: [2023-04-29 09:45:07,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +42: [2023-04-29 09:45:07,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +28: [2023-04-29 09:45:07,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +28: [2023-04-29 09:45:07,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +28: [2023-04-29 09:45:07,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +43: [2023-04-29 09:45:07,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +28: [2023-04-29 09:45:07,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +22: [2023-04-29 09:45:07,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +42: [2023-04-29 09:45:07,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +43: [2023-04-29 09:45:07,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +21: [2023-04-29 09:45:07,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +21: [2023-04-29 09:45:07,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +21: [2023-04-29 09:45:07,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +21: [2023-04-29 09:45:07,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +42: [2023-04-29 09:45:07,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +33: [2023-04-29 09:45:07,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +17: [2023-04-29 09:45:07,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +30: [2023-04-29 09:45:07,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +43: [2023-04-29 09:45:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +58: [2023-04-29 09:45:07,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +58: [2023-04-29 09:45:07,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +13: [2023-04-29 09:45:07,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +58: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +14: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +14: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +57: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +13: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +58: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +13: [2023-04-29 09:45:07,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +53: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +53: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +57: [2023-04-29 09:45:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +46: [2023-04-29 09:45:07,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +22: [2023-04-29 09:45:07,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +12: [2023-04-29 09:45:07,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +41: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +22: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +57: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +14: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +14: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +53: [2023-04-29 09:45:07,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +53: [2023-04-29 09:45:07,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +55: [2023-04-29 09:45:07,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +22: [2023-04-29 09:45:07,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +12: [2023-04-29 09:45:07,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +41: [2023-04-29 09:45:07,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +55: [2023-04-29 09:45:07,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +12: [2023-04-29 09:45:07,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +55: [2023-04-29 09:45:07,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +33: [2023-04-29 09:45:07,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +41: [2023-04-29 09:45:07,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +43: [2023-04-29 09:45:07,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +43: [2023-04-29 09:45:07,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +21: [2023-04-29 09:45:07,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +21: [2023-04-29 09:45:07,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:07,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +33: [2023-04-29 09:45:07,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +28: [2023-04-29 09:45:07,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +21: [2023-04-29 09:45:07,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +43: [2023-04-29 09:45:07,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +22: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +10: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +10: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +10: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +28: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +15: [2023-04-29 09:45:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +31: [2023-04-29 09:45:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +31: [2023-04-29 09:45:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +28: [2023-04-29 09:45:07,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:07,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +22: [2023-04-29 09:45:07,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:07,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +26: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +26: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +26: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +12: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +22: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +31: [2023-04-29 09:45:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. + 3: [2023-04-29 09:45:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +14: [2023-04-29 09:45:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +31: [2023-04-29 09:45:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +31: [2023-04-29 09:45:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +43: [2023-04-29 09:45:07,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +12: [2023-04-29 09:45:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +31: [2023-04-29 09:45:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +12: [2023-04-29 09:45:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +29: [2023-04-29 09:45:07,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +29: [2023-04-29 09:45:07,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +29: [2023-04-29 09:45:07,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +29: [2023-04-29 09:45:07,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +17: [2023-04-29 09:45:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:07,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:07,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:07,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:07,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:07,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:07,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:07,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +39: [2023-04-29 09:45:07,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +39: [2023-04-29 09:45:07,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +15: [2023-04-29 09:45:07,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +18: [2023-04-29 09:45:07,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:07,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:07,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:07,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +26: [2023-04-29 09:45:07,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +39: [2023-04-29 09:45:07,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +39: [2023-04-29 09:45:07,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +26: [2023-04-29 09:45:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +10: [2023-04-29 09:45:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +58: [2023-04-29 09:45:07,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +29: [2023-04-29 09:45:07,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +11: [2023-04-29 09:45:07,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +11: [2023-04-29 09:45:07,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +11: [2023-04-29 09:45:07,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +58: [2023-04-29 09:45:07,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +13: [2023-04-29 09:45:07,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +58: [2023-04-29 09:45:07,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +33: [2023-04-29 09:45:07,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +33: [2023-04-29 09:45:07,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +33: [2023-04-29 09:45:07,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +58: [2023-04-29 09:45:07,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +46: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:07,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:07,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +29: [2023-04-29 09:45:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +46: [2023-04-29 09:45:07,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +53: [2023-04-29 09:45:07,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +57: [2023-04-29 09:45:07,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +58: [2023-04-29 09:45:07,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +53: [2023-04-29 09:45:07,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +58: [2023-04-29 09:45:07,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +53: [2023-04-29 09:45:07,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +41: [2023-04-29 09:45:07,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +46: [2023-04-29 09:45:07,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +58: [2023-04-29 09:45:07,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +46: [2023-04-29 09:45:07,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +46: [2023-04-29 09:45:07,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +13: [2023-04-29 09:45:07,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +57: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +57: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +57: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +13: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +41: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +55: [2023-04-29 09:45:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +57: [2023-04-29 09:45:07,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +53: [2023-04-29 09:45:07,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +58: [2023-04-29 09:45:07,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +21: [2023-04-29 09:45:07,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +53: [2023-04-29 09:45:07,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +21: [2023-04-29 09:45:07,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +41: [2023-04-29 09:45:07,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +21: [2023-04-29 09:45:07,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +55: [2023-04-29 09:45:07,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +55: [2023-04-29 09:45:07,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +55: [2023-04-29 09:45:07,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +55: [2023-04-29 09:45:07,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +39: [2023-04-29 09:45:07,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +41: [2023-04-29 09:45:07,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +39: [2023-04-29 09:45:07,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +21: [2023-04-29 09:45:07,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +11: [2023-04-29 09:45:07,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +21: [2023-04-29 09:45:07,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +39: [2023-04-29 09:45:07,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +31: [2023-04-29 09:45:07,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +31: [2023-04-29 09:45:07,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +13: [2023-04-29 09:45:07,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +31: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +31: [2023-04-29 09:45:07,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +54: [2023-04-29 09:45:07,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +54: [2023-04-29 09:45:07,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +29: [2023-04-29 09:45:07,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +41: [2023-04-29 09:45:07,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +41: [2023-04-29 09:45:07,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +21: [2023-04-29 09:45:07,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +21: [2023-04-29 09:45:07,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:07,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +33: [2023-04-29 09:45:07,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +13: [2023-04-29 09:45:07,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +13: [2023-04-29 09:45:07,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:07,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +13: [2023-04-29 09:45:07,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:07,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +13: [2023-04-29 09:45:07,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +54: [2023-04-29 09:45:07,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +28: [2023-04-29 09:45:07,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +28: [2023-04-29 09:45:07,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +28: [2023-04-29 09:45:07,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +14: [2023-04-29 09:45:07,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +14: [2023-04-29 09:45:07,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +28: [2023-04-29 09:45:07,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +33: [2023-04-29 09:45:07,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +17: [2023-04-29 09:45:07,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +28: [2023-04-29 09:45:07,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +28: [2023-04-29 09:45:07,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +28: [2023-04-29 09:45:07,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +17: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +31: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:07,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +31: [2023-04-29 09:45:07,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:07,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +28: [2023-04-29 09:45:07,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +14: [2023-04-29 09:45:07,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +19: [2023-04-29 09:45:07,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:07,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +19: [2023-04-29 09:45:07,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +18: [2023-04-29 09:45:07,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +18: [2023-04-29 09:45:07,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +18: [2023-04-29 09:45:07,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +31: [2023-04-29 09:45:07,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:07,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:07,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +18: [2023-04-29 09:45:07,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:07,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +18: [2023-04-29 09:45:07,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:07,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +54: [2023-04-29 09:45:07,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +26: [2023-04-29 09:45:07,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +10: [2023-04-29 09:45:07,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +15: [2023-04-29 09:45:07,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +54: [2023-04-29 09:45:07,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +10: [2023-04-29 09:45:07,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +31: [2023-04-29 09:45:07,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +31: [2023-04-29 09:45:07,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +15: [2023-04-29 09:45:07,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +26: [2023-04-29 09:45:07,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +54: [2023-04-29 09:45:07,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +54: [2023-04-29 09:45:07,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +26: [2023-04-29 09:45:07,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:07,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +23: [2023-04-29 09:45:07,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +10: [2023-04-29 09:45:07,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +23: [2023-04-29 09:45:07,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +15: [2023-04-29 09:45:07,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +23: [2023-04-29 09:45:07,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +26: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:07,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:07,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +10: [2023-04-29 09:45:07,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +10: [2023-04-29 09:45:07,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +23: [2023-04-29 09:45:07,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +17: [2023-04-29 09:45:07,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +14: [2023-04-29 09:45:07,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:07,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +17: [2023-04-29 09:45:07,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +31: [2023-04-29 09:45:07,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:07,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:07,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +14: [2023-04-29 09:45:07,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +31: [2023-04-29 09:45:07,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +29: [2023-04-29 09:45:07,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +29: [2023-04-29 09:45:07,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +11: [2023-04-29 09:45:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +29: [2023-04-29 09:45:07,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +18: [2023-04-29 09:45:07,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +18: [2023-04-29 09:45:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +11: [2023-04-29 09:45:07,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +11: [2023-04-29 09:45:07,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +11: [2023-04-29 09:45:07,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +11: [2023-04-29 09:45:07,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +23: [2023-04-29 09:45:07,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:07,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +33: [2023-04-29 09:45:07,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +18: [2023-04-29 09:45:07,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +30: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +30: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +18: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +18: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:07,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +25: [2023-04-29 09:45:07,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +30: [2023-04-29 09:45:07,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +23: [2023-04-29 09:45:07,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_05-model_01-model_states.pt. +45: [2023-04-29 09:45:07,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +45: [2023-04-29 09:45:07,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +18: [2023-04-29 09:45:07,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +45: [2023-04-29 09:45:07,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +45: [2023-04-29 09:45:07,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +33: [2023-04-29 09:45:07,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +25: [2023-04-29 09:45:07,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +25: [2023-04-29 09:45:07,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +61: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +61: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +61: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +18: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +61: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +11: [2023-04-29 09:45:07,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:07,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +28: [2023-04-29 09:45:07,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +33: [2023-04-29 09:45:07,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +28: [2023-04-29 09:45:07,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +44: [2023-04-29 09:45:07,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +44: [2023-04-29 09:45:07,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +33: [2023-04-29 09:45:07,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +33: [2023-04-29 09:45:07,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +28: [2023-04-29 09:45:07,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:07,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +44: [2023-04-29 09:45:07,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +27: [2023-04-29 09:45:07,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +44: [2023-04-29 09:45:07,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +24: [2023-04-29 09:45:07,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +27: [2023-04-29 09:45:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +48: [2023-04-29 09:45:07,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +15: [2023-04-29 09:45:07,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +48: [2023-04-29 09:45:07,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 8: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +33: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +24: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +33: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +19: [2023-04-29 09:45:07,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +24: [2023-04-29 09:45:07,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +30: [2023-04-29 09:45:07,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +19: [2023-04-29 09:45:07,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +30: [2023-04-29 09:45:07,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +31: [2023-04-29 09:45:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +45: [2023-04-29 09:45:07,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +45: [2023-04-29 09:45:07,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +45: [2023-04-29 09:45:07,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +45: [2023-04-29 09:45:07,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +31: [2023-04-29 09:45:07,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +30: [2023-04-29 09:45:07,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt... +61: [2023-04-29 09:45:07,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:07,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:07,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +61: [2023-04-29 09:45:07,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +31: [2023-04-29 09:45:07,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +31: [2023-04-29 09:45:07,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +31: [2023-04-29 09:45:07,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +25: [2023-04-29 09:45:07,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +14: [2023-04-29 09:45:07,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:07,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +31: [2023-04-29 09:45:07,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +13: [2023-04-29 09:45:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +25: [2023-04-29 09:45:07,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +37: [2023-04-29 09:45:07,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +37: [2023-04-29 09:45:07,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +44: [2023-04-29 09:45:07,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:07,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +44: [2023-04-29 09:45:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +44: [2023-04-29 09:45:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +27: [2023-04-29 09:45:07,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:07,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +37: [2023-04-29 09:45:07,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:07,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:07,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:07,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +37: [2023-04-29 09:45:07,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +31: [2023-04-29 09:45:07,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +48: [2023-04-29 09:45:07,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +14: [2023-04-29 09:45:07,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:07,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:07,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:07,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:07,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:07,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +48: [2023-04-29 09:45:07,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +20: [2023-04-29 09:45:07,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:07,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +20: [2023-04-29 09:45:07,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +20: [2023-04-29 09:45:07,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:07,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +19: [2023-04-29 09:45:07,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +20: [2023-04-29 09:45:07,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:07,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +23: [2023-04-29 09:45:07,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +20: [2023-04-29 09:45:07,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +39: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +10: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +10: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +10: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +10: [2023-04-29 09:45:07,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:07,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +34: [2023-04-29 09:45:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +23: [2023-04-29 09:45:07,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +37: [2023-04-29 09:45:07,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +19: [2023-04-29 09:45:07,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +39: [2023-04-29 09:45:07,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +23: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +37: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +19: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +16: [2023-04-29 09:45:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +48: [2023-04-29 09:45:07,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +16: [2023-04-29 09:45:07,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:07,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +37: [2023-04-29 09:45:07,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +10: [2023-04-29 09:45:07,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +39: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +34: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +10: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +39: [2023-04-29 09:45:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +34: [2023-04-29 09:45:07,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +34: [2023-04-29 09:45:07,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +39: [2023-04-29 09:45:07,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +34: [2023-04-29 09:45:07,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +39: [2023-04-29 09:45:07,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +39: [2023-04-29 09:45:07,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +18: [2023-04-29 09:45:07,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:07,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +18: [2023-04-29 09:45:07,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +48: [2023-04-29 09:45:07,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +48: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +58: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +16: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +58: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +16: [2023-04-29 09:45:07,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:07,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +16: [2023-04-29 09:45:07,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:07,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:07,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +58: [2023-04-29 09:45:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +23: [2023-04-29 09:45:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +58: [2023-04-29 09:45:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +58: [2023-04-29 09:45:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +18: [2023-04-29 09:45:07,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +28: [2023-04-29 09:45:07,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:07,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +24: [2023-04-29 09:45:07,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:07,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +63: [2023-04-29 09:45:07,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +63: [2023-04-29 09:45:07,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +14: [2023-04-29 09:45:07,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +23: [2023-04-29 09:45:07,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:07,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:07,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +23: [2023-04-29 09:45:07,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +28: [2023-04-29 09:45:07,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +45: [2023-04-29 09:45:07,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +24: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +45: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +23: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +61: [2023-04-29 09:45:07,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +23: [2023-04-29 09:45:07,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +47: [2023-04-29 09:45:07,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +47: [2023-04-29 09:45:07,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +47: [2023-04-29 09:45:07,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +30: [2023-04-29 09:45:07,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +47: [2023-04-29 09:45:07,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_27-model_01-model_states.pt. +23: [2023-04-29 09:45:07,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +30: [2023-04-29 09:45:07,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +45: [2023-04-29 09:45:07,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +30: [2023-04-29 09:45:07,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +23: [2023-04-29 09:45:07,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +45: [2023-04-29 09:45:07,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +45: [2023-04-29 09:45:07,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +23: [2023-04-29 09:45:07,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +30: [2023-04-29 09:45:07,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_00-model_states.pt. +61: [2023-04-29 09:45:07,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +28: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +44: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +45: [2023-04-29 09:45:07,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:07,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:07,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +61: [2023-04-29 09:45:07,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +61: [2023-04-29 09:45:07,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +61: [2023-04-29 09:45:07,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +44: [2023-04-29 09:45:07,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +44: [2023-04-29 09:45:07,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +30: [2023-04-29 09:45:07,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt... +27: [2023-04-29 09:45:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +27: [2023-04-29 09:45:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +27: [2023-04-29 09:45:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +25: [2023-04-29 09:45:07,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +27: [2023-04-29 09:45:07,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +44: [2023-04-29 09:45:07,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +45: [2023-04-29 09:45:07,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +28: [2023-04-29 09:45:07,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:07,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:07,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:07,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +45: [2023-04-29 09:45:07,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +45: [2023-04-29 09:45:07,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +45: [2023-04-29 09:45:07,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +24: [2023-04-29 09:45:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +20: [2023-04-29 09:45:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +44: [2023-04-29 09:45:07,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +44: [2023-04-29 09:45:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +25: [2023-04-29 09:45:07,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +25: [2023-04-29 09:45:07,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +44: [2023-04-29 09:45:07,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 4: [2023-04-29 09:45:07,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +63: [2023-04-29 09:45:07,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +37: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +25: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:07,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +20: [2023-04-29 09:45:07,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +20: [2023-04-29 09:45:07,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +20: [2023-04-29 09:45:07,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +20: [2023-04-29 09:45:07,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +25: [2023-04-29 09:45:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 4: [2023-04-29 09:45:07,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:07,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +37: [2023-04-29 09:45:07,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +37: [2023-04-29 09:45:07,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +37: [2023-04-29 09:45:07,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +37: [2023-04-29 09:45:07,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +28: [2023-04-29 09:45:07,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:07,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +47: [2023-04-29 09:45:07,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +47: [2023-04-29 09:45:07,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +39: [2023-04-29 09:45:07,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +25: [2023-04-29 09:45:07,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +25: [2023-04-29 09:45:07,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +47: [2023-04-29 09:45:07,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +50: [2023-04-29 09:45:07,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +50: [2023-04-29 09:45:07,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +50: [2023-04-29 09:45:07,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +50: [2023-04-29 09:45:07,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 4: [2023-04-29 09:45:07,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +47: [2023-04-29 09:45:07,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt... +25: [2023-04-29 09:45:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +24: [2023-04-29 09:45:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +24: [2023-04-29 09:45:07,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:07,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:07,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +37: [2023-04-29 09:45:07,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +24: [2023-04-29 09:45:07,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +16: [2023-04-29 09:45:07,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +10: [2023-04-29 09:45:07,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +48: [2023-04-29 09:45:07,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +27: [2023-04-29 09:45:07,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +24: [2023-04-29 09:45:07,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +37: [2023-04-29 09:45:07,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +48: [2023-04-29 09:45:07,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +20: [2023-04-29 09:45:07,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +34: [2023-04-29 09:45:07,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +27: [2023-04-29 09:45:07,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +39: [2023-04-29 09:45:07,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +20: [2023-04-29 09:45:07,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:07,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:07,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +37: [2023-04-29 09:45:07,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +48: [2023-04-29 09:45:07,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +27: [2023-04-29 09:45:07,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +27: [2023-04-29 09:45:07,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +20: [2023-04-29 09:45:07,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +58: [2023-04-29 09:45:07,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +10: [2023-04-29 09:45:07,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +10: [2023-04-29 09:45:07,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +10: [2023-04-29 09:45:07,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +24: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +37: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 3: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +58: [2023-04-29 09:45:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 3: [2023-04-29 09:45:07,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +39: [2023-04-29 09:45:07,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:07,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +58: [2023-04-29 09:45:07,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +20: [2023-04-29 09:45:07,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +50: [2023-04-29 09:45:07,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +16: [2023-04-29 09:45:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +37: [2023-04-29 09:45:07,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +16: [2023-04-29 09:45:07,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:07,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +36: [2023-04-29 09:45:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +36: [2023-04-29 09:45:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +36: [2023-04-29 09:45:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +20: [2023-04-29 09:45:07,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +50: [2023-04-29 09:45:07,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +20: [2023-04-29 09:45:07,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +50: [2023-04-29 09:45:07,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +48: [2023-04-29 09:45:07,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +10: [2023-04-29 09:45:07,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:07,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +34: [2023-04-29 09:45:07,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +16: [2023-04-29 09:45:07,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:07,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:07,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +16: [2023-04-29 09:45:07,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +39: [2023-04-29 09:45:07,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:07,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +48: [2023-04-29 09:45:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +25: [2023-04-29 09:45:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +48: [2023-04-29 09:45:07,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +10: [2023-04-29 09:45:07,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +20: [2023-04-29 09:45:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +20: [2023-04-29 09:45:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:07,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:07,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +34: [2023-04-29 09:45:07,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +58: [2023-04-29 09:45:07,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +34: [2023-04-29 09:45:07,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:07,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +58: [2023-04-29 09:45:07,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:07,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +59: [2023-04-29 09:45:07,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +59: [2023-04-29 09:45:07,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +59: [2023-04-29 09:45:07,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +62: [2023-04-29 09:45:07,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +62: [2023-04-29 09:45:07,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +62: [2023-04-29 09:45:07,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +62: [2023-04-29 09:45:07,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +36: [2023-04-29 09:45:07,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:07,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +36: [2023-04-29 09:45:07,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +20: [2023-04-29 09:45:07,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:07,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:07,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +37: [2023-04-29 09:45:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +36: [2023-04-29 09:45:07,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +36: [2023-04-29 09:45:07,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:07,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:07,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +49: [2023-04-29 09:45:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +49: [2023-04-29 09:45:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +49: [2023-04-29 09:45:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +49: [2023-04-29 09:45:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +25: [2023-04-29 09:45:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +25: [2023-04-29 09:45:07,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +47: [2023-04-29 09:45:07,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +24: [2023-04-29 09:45:07,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +37: [2023-04-29 09:45:07,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:07,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +19: [2023-04-29 09:45:07,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +19: [2023-04-29 09:45:07,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +19: [2023-04-29 09:45:07,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +19: [2023-04-29 09:45:07,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +47: [2023-04-29 09:45:07,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +27: [2023-04-29 09:45:07,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +47: [2023-04-29 09:45:07,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_00-model_states.pt. +24: [2023-04-29 09:45:07,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +47: [2023-04-29 09:45:07,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +25: [2023-04-29 09:45:07,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:07,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +27: [2023-04-29 09:45:07,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +47: [2023-04-29 09:45:07,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +59: [2023-04-29 09:45:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +47: [2023-04-29 09:45:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt... +24: [2023-04-29 09:45:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +24: [2023-04-29 09:45:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +62: [2023-04-29 09:45:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +27: [2023-04-29 09:45:07,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +45: [2023-04-29 09:45:07,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:07,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +21: [2023-04-29 09:45:07,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +21: [2023-04-29 09:45:07,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +21: [2023-04-29 09:45:07,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +21: [2023-04-29 09:45:07,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +62: [2023-04-29 09:45:07,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +20: [2023-04-29 09:45:07,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +60: [2023-04-29 09:45:07,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +60: [2023-04-29 09:45:07,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +60: [2023-04-29 09:45:07,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +60: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +54: [2023-04-29 09:45:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +27: [2023-04-29 09:45:07,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +54: [2023-04-29 09:45:07,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +62: [2023-04-29 09:45:07,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +37: [2023-04-29 09:45:07,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:07,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +24: [2023-04-29 09:45:07,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +34: [2023-04-29 09:45:07,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +25: [2023-04-29 09:45:07,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:07,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +34: [2023-04-29 09:45:07,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +24: [2023-04-29 09:45:07,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +27: [2023-04-29 09:45:07,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +49: [2023-04-29 09:45:07,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +20: [2023-04-29 09:45:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +27: [2023-04-29 09:45:07,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +60: [2023-04-29 09:45:07,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +54: [2023-04-29 09:45:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +54: [2023-04-29 09:45:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +54: [2023-04-29 09:45:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +54: [2023-04-29 09:45:07,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +19: [2023-04-29 09:45:07,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +49: [2023-04-29 09:45:07,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +54: [2023-04-29 09:45:07,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +19: [2023-04-29 09:45:07,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:07,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:07,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +54: [2023-04-29 09:45:07,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +54: [2023-04-29 09:45:07,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +24: [2023-04-29 09:45:07,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +27: [2023-04-29 09:45:07,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +54: [2023-04-29 09:45:07,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +49: [2023-04-29 09:45:07,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +27: [2023-04-29 09:45:07,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +49: [2023-04-29 09:45:07,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +37: [2023-04-29 09:45:07,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +26: [2023-04-29 09:45:07,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +37: [2023-04-29 09:45:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:07,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +26: [2023-04-29 09:45:07,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:07,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +12: [2023-04-29 09:45:07,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +12: [2023-04-29 09:45:07,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +26: [2023-04-29 09:45:07,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:07,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +36: [2023-04-29 09:45:07,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +36: [2023-04-29 09:45:07,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +36: [2023-04-29 09:45:07,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +36: [2023-04-29 09:45:07,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:07,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +34: [2023-04-29 09:45:07,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +16: [2023-04-29 09:45:07,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +16: [2023-04-29 09:45:07,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +16: [2023-04-29 09:45:07,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +16: [2023-04-29 09:45:07,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +36: [2023-04-29 09:45:07,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +20: [2023-04-29 09:45:07,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +34: [2023-04-29 09:45:07,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +21: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +11: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +20: [2023-04-29 09:45:07,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +20: [2023-04-29 09:45:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +42: [2023-04-29 09:45:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +42: [2023-04-29 09:45:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +36: [2023-04-29 09:45:07,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +21: [2023-04-29 09:45:07,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 3: [2023-04-29 09:45:07,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +42: [2023-04-29 09:45:07,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +42: [2023-04-29 09:45:07,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 3: [2023-04-29 09:45:07,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +36: [2023-04-29 09:45:07,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +20: [2023-04-29 09:45:07,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 1: [2023-04-29 09:45:07,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:07,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +12: [2023-04-29 09:45:07,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:07,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +36: [2023-04-29 09:45:07,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +12: [2023-04-29 09:45:07,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +32: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +12: [2023-04-29 09:45:07,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +32: [2023-04-29 09:45:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:07,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:07,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +36: [2023-04-29 09:45:07,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +40: [2023-04-29 09:45:07,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +40: [2023-04-29 09:45:07,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +40: [2023-04-29 09:45:07,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +36: [2023-04-29 09:45:07,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +32: [2023-04-29 09:45:07,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:07,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +16: [2023-04-29 09:45:07,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +36: [2023-04-29 09:45:07,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +16: [2023-04-29 09:45:07,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +36: [2023-04-29 09:45:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +42: [2023-04-29 09:45:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +42: [2023-04-29 09:45:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +42: [2023-04-29 09:45:07,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +21: [2023-04-29 09:45:07,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +53: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +10: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +21: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +10: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +28: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +28: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +53: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +36: [2023-04-29 09:45:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +43: [2023-04-29 09:45:07,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +52: [2023-04-29 09:45:07,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +52: [2023-04-29 09:45:07,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:07,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +28: [2023-04-29 09:45:07,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 1: [2023-04-29 09:45:07,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:07,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +52: [2023-04-29 09:45:07,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +52: [2023-04-29 09:45:07,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +54: [2023-04-29 09:45:07,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 1: [2023-04-29 09:45:07,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:07,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +15: [2023-04-29 09:45:07,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +15: [2023-04-29 09:45:07,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +54: [2023-04-29 09:45:07,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +15: [2023-04-29 09:45:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +22: [2023-04-29 09:45:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +22: [2023-04-29 09:45:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +21: [2023-04-29 09:45:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:07,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 3: [2023-04-29 09:45:07,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +60: [2023-04-29 09:45:07,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +19: [2023-04-29 09:45:07,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +60: [2023-04-29 09:45:07,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +22: [2023-04-29 09:45:07,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +21: [2023-04-29 09:45:07,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +40: [2023-04-29 09:45:07,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +19: [2023-04-29 09:45:07,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +34: [2023-04-29 09:45:07,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +34: [2023-04-29 09:45:07,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +36: [2023-04-29 09:45:07,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +10: [2023-04-29 09:45:07,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 9: [2023-04-29 09:45:07,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +43: [2023-04-29 09:45:07,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +26: [2023-04-29 09:45:07,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +60: [2023-04-29 09:45:07,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +10: [2023-04-29 09:45:07,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +56: [2023-04-29 09:45:07,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +56: [2023-04-29 09:45:07,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:07,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +56: [2023-04-29 09:45:07,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +29: [2023-04-29 09:45:07,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +34: [2023-04-29 09:45:07,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +43: [2023-04-29 09:45:07,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +10: [2023-04-29 09:45:07,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +29: [2023-04-29 09:45:07,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +54: [2023-04-29 09:45:07,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +29: [2023-04-29 09:45:07,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +54: [2023-04-29 09:45:07,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +34: [2023-04-29 09:45:07,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +38: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +35: [2023-04-29 09:45:07,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +38: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +35: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +19: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +38: [2023-04-29 09:45:07,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +34: [2023-04-29 09:45:07,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +36: [2023-04-29 09:45:07,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +21: [2023-04-29 09:45:07,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +52: [2023-04-29 09:45:07,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +28: [2023-04-29 09:45:07,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:07,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +35: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +28: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +28: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +15: [2023-04-29 09:45:07,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +34: [2023-04-29 09:45:07,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +15: [2023-04-29 09:45:07,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +35: [2023-04-29 09:45:07,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +60: [2023-04-29 09:45:07,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +36: [2023-04-29 09:45:07,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +19: [2023-04-29 09:45:07,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +34: [2023-04-29 09:45:07,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +36: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +15: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +60: [2023-04-29 09:45:07,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:07,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +22: [2023-04-29 09:45:07,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +54: [2023-04-29 09:45:07,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +54: [2023-04-29 09:45:07,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +22: [2023-04-29 09:45:07,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +22: [2023-04-29 09:45:07,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +34: [2023-04-29 09:45:07,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +22: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +32: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +32: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +32: [2023-04-29 09:45:07,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +11: [2023-04-29 09:45:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +11: [2023-04-29 09:45:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +32: [2023-04-29 09:45:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +22: [2023-04-29 09:45:07,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +11: [2023-04-29 09:45:07,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 0: [2023-04-29 09:45:07,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 8: [2023-04-29 09:45:07,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 8: [2023-04-29 09:45:07,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +21: [2023-04-29 09:45:07,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:07,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:07,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +22: [2023-04-29 09:45:07,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:07,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:07,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +22: [2023-04-29 09:45:07,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +30: [2023-04-29 09:45:07,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +22: [2023-04-29 09:45:07,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:07,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +55: [2023-04-29 09:45:07,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +55: [2023-04-29 09:45:07,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +55: [2023-04-29 09:45:07,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +55: [2023-04-29 09:45:07,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +16: [2023-04-29 09:45:07,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +36: [2023-04-29 09:45:07,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +12: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +30: [2023-04-29 09:45:07,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +56: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +56: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +35: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +12: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +16: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +30: [2023-04-29 09:45:07,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +56: [2023-04-29 09:45:07,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +56: [2023-04-29 09:45:07,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +13: [2023-04-29 09:45:07,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +13: [2023-04-29 09:45:07,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +12: [2023-04-29 09:45:07,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +38: [2023-04-29 09:45:07,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +26: [2023-04-29 09:45:07,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:07,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:07,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +12: [2023-04-29 09:45:07,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +38: [2023-04-29 09:45:07,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +43: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +43: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +35: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +12: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +32: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +12: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +32: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +38: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +29: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +43: [2023-04-29 09:45:07,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:07,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +13: [2023-04-29 09:45:07,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +53: [2023-04-29 09:45:07,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +43: [2023-04-29 09:45:07,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +16: [2023-04-29 09:45:07,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +16: [2023-04-29 09:45:07,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +53: [2023-04-29 09:45:07,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +36: [2023-04-29 09:45:07,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +54: [2023-04-29 09:45:07,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +29: [2023-04-29 09:45:07,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +14: [2023-04-29 09:45:07,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +54: [2023-04-29 09:45:07,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +36: [2023-04-29 09:45:07,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:07,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +14: [2023-04-29 09:45:07,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +16: [2023-04-29 09:45:07,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:07,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +36: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +29: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +43: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +11: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +21: [2023-04-29 09:45:07,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +32: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +16: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +16: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 2: [2023-04-29 09:45:07,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +11: [2023-04-29 09:45:07,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +32: [2023-04-29 09:45:07,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +32: [2023-04-29 09:45:07,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +16: [2023-04-29 09:45:07,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:07,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +17: [2023-04-29 09:45:07,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +17: [2023-04-29 09:45:07,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 2: [2023-04-29 09:45:07,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +21: [2023-04-29 09:45:07,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +11: [2023-04-29 09:45:07,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +41: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +41: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +41: [2023-04-29 09:45:07,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +29: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +55: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:07,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +55: [2023-04-29 09:45:07,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +26: [2023-04-29 09:45:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +26: [2023-04-29 09:45:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +26: [2023-04-29 09:45:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +17: [2023-04-29 09:45:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +26: [2023-04-29 09:45:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +17: [2023-04-29 09:45:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +55: [2023-04-29 09:45:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +35: [2023-04-29 09:45:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +11: [2023-04-29 09:45:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:07,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +55: [2023-04-29 09:45:07,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +21: [2023-04-29 09:45:07,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +41: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +41: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +41: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +51: [2023-04-29 09:45:07,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +51: [2023-04-29 09:45:07,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +51: [2023-04-29 09:45:07,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +13: [2023-04-29 09:45:07,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +51: [2023-04-29 09:45:07,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +23: [2023-04-29 09:45:07,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +23: [2023-04-29 09:45:07,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +21: [2023-04-29 09:45:07,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +12: [2023-04-29 09:45:07,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +23: [2023-04-29 09:45:07,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +10: [2023-04-29 09:45:07,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +12: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +43: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +14: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +61: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +25: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +25: [2023-04-29 09:45:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +22: [2023-04-29 09:45:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +18: [2023-04-29 09:45:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +18: [2023-04-29 09:45:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +18: [2023-04-29 09:45:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +18: [2023-04-29 09:45:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +32: [2023-04-29 09:45:07,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:07,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +32: [2023-04-29 09:45:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +10: [2023-04-29 09:45:07,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +43: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +22: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +53: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +63: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +14: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +25: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +63: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:07,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +61: [2023-04-29 09:45:07,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +58: [2023-04-29 09:45:07,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +58: [2023-04-29 09:45:07,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +14: [2023-04-29 09:45:07,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +10: [2023-04-29 09:45:07,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +10: [2023-04-29 09:45:07,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +10: [2023-04-29 09:45:07,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +57: [2023-04-29 09:45:07,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +57: [2023-04-29 09:45:07,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +14: [2023-04-29 09:45:07,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +57: [2023-04-29 09:45:07,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +17: [2023-04-29 09:45:07,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +63: [2023-04-29 09:45:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +63: [2023-04-29 09:45:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +63: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +19: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +19: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +30: [2023-04-29 09:45:07,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +57: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +61: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +61: [2023-04-29 09:45:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +44: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +44: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +61: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +61: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +57: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +57: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +35: [2023-04-29 09:45:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +10: [2023-04-29 09:45:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +15: [2023-04-29 09:45:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +44: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +44: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +30: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +27: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +27: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +57: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +15: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +15: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +15: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +27: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +27: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +22: [2023-04-29 09:45:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +22: [2023-04-29 09:45:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +43: [2023-04-29 09:45:07,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +26: [2023-04-29 09:45:07,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +27: [2023-04-29 09:45:07,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:07,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +43: [2023-04-29 09:45:07,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +43: [2023-04-29 09:45:07,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +10: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +27: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +27: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:07,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:07,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:07,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:07,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +23: [2023-04-29 09:45:07,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +28: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +28: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +28: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +26: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +22: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +22: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +22: [2023-04-29 09:45:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +28: [2023-04-29 09:45:07,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +43: [2023-04-29 09:45:07,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +22: [2023-04-29 09:45:07,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +25: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +43: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +25: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +25: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +52: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +43: [2023-04-29 09:45:07,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +22: [2023-04-29 09:45:07,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +58: [2023-04-29 09:45:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +18: [2023-04-29 09:45:07,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +58: [2023-04-29 09:45:07,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +28: [2023-04-29 09:45:07,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +22: [2023-04-29 09:45:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +32: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +52: [2023-04-29 09:45:07,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +52: [2023-04-29 09:45:07,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +25: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +35: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +57: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +52: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +52: [2023-04-29 09:45:07,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +11: [2023-04-29 09:45:07,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +32: [2023-04-29 09:45:07,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +43: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +29: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +29: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +44: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +13: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +13: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +44: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +44: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +44: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 9: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +61: [2023-04-29 09:45:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +57: [2023-04-29 09:45:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +11: [2023-04-29 09:45:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 3: [2023-04-29 09:45:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +57: [2023-04-29 09:45:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +32: [2023-04-29 09:45:07,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +33: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +30: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +30: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +22: [2023-04-29 09:45:07,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +61: [2023-04-29 09:45:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +29: [2023-04-29 09:45:07,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +32: [2023-04-29 09:45:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +11: [2023-04-29 09:45:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +32: [2023-04-29 09:45:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +30: [2023-04-29 09:45:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +51: [2023-04-29 09:45:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +33: [2023-04-29 09:45:07,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +29: [2023-04-29 09:45:07,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +51: [2023-04-29 09:45:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +51: [2023-04-29 09:45:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 4: [2023-04-29 09:45:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +11: [2023-04-29 09:45:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +51: [2023-04-29 09:45:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +51: [2023-04-29 09:45:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +32: [2023-04-29 09:45:07,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +41: [2023-04-29 09:45:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +41: [2023-04-29 09:45:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +33: [2023-04-29 09:45:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +11: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +60: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +60: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +53: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +46: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +46: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. + 5: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +53: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +53: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +43: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +11: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +53: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +41: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +60: [2023-04-29 09:45:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +46: [2023-04-29 09:45:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +22: [2023-04-29 09:45:07,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +41: [2023-04-29 09:45:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +41: [2023-04-29 09:45:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +29: [2023-04-29 09:45:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +35: [2023-04-29 09:45:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +32: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +31: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +31: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +38: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +29: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +11: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +46: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +49: [2023-04-29 09:45:07,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +31: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +31: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +33: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +38: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +38: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +13: [2023-04-29 09:45:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +59: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +41: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +22: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +59: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 5: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +59: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +13: [2023-04-29 09:45:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +59: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +14: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +17: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +49: [2023-04-29 09:45:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +49: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +29: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +38: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +31: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +38: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +40: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +31: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +29: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +31: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +35: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +47: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +35: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +32: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +17: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +50: [2023-04-29 09:45:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +17: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +50: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +13: [2023-04-29 09:45:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +38: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +38: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +38: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +13: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +38: [2023-04-29 09:45:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +43: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +13: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +40: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +17: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +47: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +47: [2023-04-29 09:45:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +49: [2023-04-29 09:45:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +17: [2023-04-29 09:45:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +40: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +40: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +40: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +51: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +51: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +63: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +49: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +42: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +59: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +49: [2023-04-29 09:45:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +59: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +40: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +13: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +59: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +50: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +62: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +17: [2023-04-29 09:45:07,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +50: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +42: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +42: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +13: [2023-04-29 09:45:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +35: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +43: [2023-04-29 09:45:07,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +50: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +50: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +62: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +62: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +50: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 8: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +42: [2023-04-29 09:45:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +51: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +18: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +51: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +42: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +63: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +63: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +35: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +35: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +13: [2023-04-29 09:45:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +42: [2023-04-29 09:45:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +42: [2023-04-29 09:45:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +46: [2023-04-29 09:45:07,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +43: [2023-04-29 09:45:07,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +18: [2023-04-29 09:45:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +42: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +51: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +14: [2023-04-29 09:45:07,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +33: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +33: [2023-04-29 09:45:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +30: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +42: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +17: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +17: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +62: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +62: [2023-04-29 09:45:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +62: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 5: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +62: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +62: [2023-04-29 09:45:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +60: [2023-04-29 09:45:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +14: [2023-04-29 09:45:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +60: [2023-04-29 09:45:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +14: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +18: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +53: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +41: [2023-04-29 09:45:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +26: [2023-04-29 09:45:07,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +41: [2023-04-29 09:45:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +41: [2023-04-29 09:45:07,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 9: [2023-04-29 09:45:07,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 2: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 2: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +17: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +17: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +58: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +51: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +18: [2023-04-29 09:45:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +35: [2023-04-29 09:45:07,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +41: [2023-04-29 09:45:07,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +26: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +26: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +41: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +61: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +51: [2023-04-29 09:45:07,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +15: [2023-04-29 09:45:07,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:07,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:07,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +56: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +41: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +56: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +58: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +57: [2023-04-29 09:45:07,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +27: [2023-04-29 09:45:07,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +56: [2023-04-29 09:45:07,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +63: [2023-04-29 09:45:07,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 2: [2023-04-29 09:45:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +26: [2023-04-29 09:45:07,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +56: [2023-04-29 09:45:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +56: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +26: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +29: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +29: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:07,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +15: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +57: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +26: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +15: [2023-04-29 09:45:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +15: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +23: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:07,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +18: [2023-04-29 09:45:07,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +61: [2023-04-29 09:45:07,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +61: [2023-04-29 09:45:07,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +27: [2023-04-29 09:45:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +27: [2023-04-29 09:45:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +52: [2023-04-29 09:45:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +26: [2023-04-29 09:45:07,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +23: [2023-04-29 09:45:07,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +25: [2023-04-29 09:45:07,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +58: [2023-04-29 09:45:07,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +51: [2023-04-29 09:45:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:07,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +43: [2023-04-29 09:45:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +26: [2023-04-29 09:45:07,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +15: [2023-04-29 09:45:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:07,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:07,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +25: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +17: [2023-04-29 09:45:07,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +23: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +57: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +23: [2023-04-29 09:45:07,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +58: [2023-04-29 09:45:07,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +23: [2023-04-29 09:45:07,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +23: [2023-04-29 09:45:07,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:07,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +13: [2023-04-29 09:45:07,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:07,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:07,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +63: [2023-04-29 09:45:07,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +63: [2023-04-29 09:45:07,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:07,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +52: [2023-04-29 09:45:07,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +30: [2023-04-29 09:45:07,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +51: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +55: [2023-04-29 09:45:07,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +25: [2023-04-29 09:45:07,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:07,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +45: [2023-04-29 09:45:07,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +45: [2023-04-29 09:45:07,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +57: [2023-04-29 09:45:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +58: [2023-04-29 09:45:07,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +57: [2023-04-29 09:45:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +25: [2023-04-29 09:45:07,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +25: [2023-04-29 09:45:07,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:07,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +45: [2023-04-29 09:45:07,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +61: [2023-04-29 09:45:07,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +49: [2023-04-29 09:45:07,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +48: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +48: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +18: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +48: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +30: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +45: [2023-04-29 09:45:07,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +45: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +47: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +47: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +13: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_28-model_01-model_states.pt. +13: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +55: [2023-04-29 09:45:07,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +55: [2023-04-29 09:45:07,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 0: [2023-04-29 09:45:07,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +61: [2023-04-29 09:45:07,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +55: [2023-04-29 09:45:07,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +55: [2023-04-29 09:45:07,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +55: [2023-04-29 09:45:07,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +55: [2023-04-29 09:45:07,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +46: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +46: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +57: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 0: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +46: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +25: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +46: [2023-04-29 09:45:07,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +45: [2023-04-29 09:45:07,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 7: [2023-04-29 09:45:07,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +45: [2023-04-29 09:45:07,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:07,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:07,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:07,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +61: [2023-04-29 09:45:07,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +61: [2023-04-29 09:45:07,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +61: [2023-04-29 09:45:07,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +31: [2023-04-29 09:45:07,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +30: [2023-04-29 09:45:07,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +27: [2023-04-29 09:45:07,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +57: [2023-04-29 09:45:07,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 7: [2023-04-29 09:45:07,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +45: [2023-04-29 09:45:07,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +46: [2023-04-29 09:45:07,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +46: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +31: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +31: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +57: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +46: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:07,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +57: [2023-04-29 09:45:07,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +27: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +27: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +31: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +31: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_06-model_01-model_states.pt. +45: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +39: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +30: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +39: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +40: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 6: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 7: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +31: [2023-04-29 09:45:07,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:07,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +30: [2023-04-29 09:45:07,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +52: [2023-04-29 09:45:07,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:07,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +50: [2023-04-29 09:45:07,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +57: [2023-04-29 09:45:07,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +57: [2023-04-29 09:45:07,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +17: [2023-04-29 09:45:07,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 4: [2023-04-29 09:45:07,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 7: [2023-04-29 09:45:07,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +53: [2023-04-29 09:45:07,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +61: [2023-04-29 09:45:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +13: [2023-04-29 09:45:07,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +33: [2023-04-29 09:45:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +27: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:07,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +30: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +59: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +52: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +19: [2023-04-29 09:45:07,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +50: [2023-04-29 09:45:07,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +50: [2023-04-29 09:45:07,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +19: [2023-04-29 09:45:07,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +46: [2023-04-29 09:45:07,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +19: [2023-04-29 09:45:07,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +61: [2023-04-29 09:45:07,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +62: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +59: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +17: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +17: [2023-04-29 09:45:07,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +61: [2023-04-29 09:45:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +45: [2023-04-29 09:45:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +53: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +46: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +33: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +42: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:07,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +51: [2023-04-29 09:45:07,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +49: [2023-04-29 09:45:07,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +61: [2023-04-29 09:45:07,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +13: [2023-04-29 09:45:07,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:07,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +47: [2023-04-29 09:45:07,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +45: [2023-04-29 09:45:07,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +40: [2023-04-29 09:45:07,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +31: [2023-04-29 09:45:07,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +13: [2023-04-29 09:45:07,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:07,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +56: [2023-04-29 09:45:07,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:07,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +60: [2023-04-29 09:45:07,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:07,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +60: [2023-04-29 09:45:07,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +50: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +62: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +31: [2023-04-29 09:45:07,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +44: [2023-04-29 09:45:07,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +46: [2023-04-29 09:45:07,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +46: [2023-04-29 09:45:07,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +60: [2023-04-29 09:45:07,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:07,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +53: [2023-04-29 09:45:07,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +31: [2023-04-29 09:45:07,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:07,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +39: [2023-04-29 09:45:07,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:07,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +60: [2023-04-29 09:45:07,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +48: [2023-04-29 09:45:07,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:07,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:07,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:07,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:07,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:07,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +31: [2023-04-29 09:45:07,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +47: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +44: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +33: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +40: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +46: [2023-04-29 09:45:07,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +44: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +44: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +44: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +31: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +44: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +56: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt... +60: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +48: [2023-04-29 09:45:07,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 6: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 6: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +62: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +53: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +41: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +41: [2023-04-29 09:45:07,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:07,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +18: [2023-04-29 09:45:07,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +62: [2023-04-29 09:45:07,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +18: [2023-04-29 09:45:07,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +23: [2023-04-29 09:45:07,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +50: [2023-04-29 09:45:07,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +18: [2023-04-29 09:45:07,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +33: [2023-04-29 09:45:07,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 6: [2023-04-29 09:45:07,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:07,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +48: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt... +49: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +20: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +18: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +60: [2023-04-29 09:45:07,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +42: [2023-04-29 09:45:07,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +50: [2023-04-29 09:45:07,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:07,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +51: [2023-04-29 09:45:07,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +33: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +17: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +41: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +20: [2023-04-29 09:45:07,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +15: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +59: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:07,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +47: [2023-04-29 09:45:07,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:07,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:07,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +50: [2023-04-29 09:45:07,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:07,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:07,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +33: [2023-04-29 09:45:07,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:07,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +33: [2023-04-29 09:45:07,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +42: [2023-04-29 09:45:07,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:07,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +60: [2023-04-29 09:45:07,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +56: [2023-04-29 09:45:07,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +38: [2023-04-29 09:45:07,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +33: [2023-04-29 09:45:07,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +51: [2023-04-29 09:45:07,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +51: [2023-04-29 09:45:07,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +59: [2023-04-29 09:45:07,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:07,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +59: [2023-04-29 09:45:07,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:07,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:07,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +49: [2023-04-29 09:45:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:07,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +23: [2023-04-29 09:45:07,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:07,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:07,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +31: [2023-04-29 09:45:07,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:07,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:07,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +40: [2023-04-29 09:45:07,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:07,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +15: [2023-04-29 09:45:07,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +23: [2023-04-29 09:45:07,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +40: [2023-04-29 09:45:07,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +49: [2023-04-29 09:45:07,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +50: [2023-04-29 09:45:07,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:07,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +56: [2023-04-29 09:45:07,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:07,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:07,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:07,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +56: [2023-04-29 09:45:07,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:07,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +55: [2023-04-29 09:45:07,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +55: [2023-04-29 09:45:07,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +20: [2023-04-29 09:45:07,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +47: [2023-04-29 09:45:07,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:07,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +15: [2023-04-29 09:45:07,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:07,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +18: [2023-04-29 09:45:07,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +20: [2023-04-29 09:45:07,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +46: [2023-04-29 09:45:07,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +46: [2023-04-29 09:45:07,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +20: [2023-04-29 09:45:07,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +62: [2023-04-29 09:45:07,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:07,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +46: [2023-04-29 09:45:07,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +18: [2023-04-29 09:45:07,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:07,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:07,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:07,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:07,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:07,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +24: [2023-04-29 09:45:08,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +56: [2023-04-29 09:45:08,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +24: [2023-04-29 09:45:08,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:08,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +37: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +37: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:08,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +45: [2023-04-29 09:45:08,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +45: [2023-04-29 09:45:08,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +24: [2023-04-29 09:45:08,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:08,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:08,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:08,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:08,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +46: [2023-04-29 09:45:08,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:08,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +45: [2023-04-29 09:45:08,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +23: [2023-04-29 09:45:08,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +58: [2023-04-29 09:45:08,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:08,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +45: [2023-04-29 09:45:08,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +27: [2023-04-29 09:45:08,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +27: [2023-04-29 09:45:08,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +27: [2023-04-29 09:45:08,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +27: [2023-04-29 09:45:08,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 6: [2023-04-29 09:45:08,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:08,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +16: [2023-04-29 09:45:08,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +16: [2023-04-29 09:45:08,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +16: [2023-04-29 09:45:08,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +45: [2023-04-29 09:45:08,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +45: [2023-04-29 09:45:08,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +20: [2023-04-29 09:45:08,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:08,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +55: [2023-04-29 09:45:08,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +46: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +55: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +45: [2023-04-29 09:45:08,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:08,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:08,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +55: [2023-04-29 09:45:08,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:08,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +55: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +24: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +24: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +24: [2023-04-29 09:45:08,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +44: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:08,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +24: [2023-04-29 09:45:08,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +31: [2023-04-29 09:45:08,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +20: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +47: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +44: [2023-04-29 09:45:08,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +44: [2023-04-29 09:45:08,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +48: [2023-04-29 09:45:08,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +48: [2023-04-29 09:45:08,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +47: [2023-04-29 09:45:08,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +48: [2023-04-29 09:45:08,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +31: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +31: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_00-model_states.pt. +19: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +19: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +19: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +19: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +20: [2023-04-29 09:45:08,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +20: [2023-04-29 09:45:08,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:08,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:08,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +20: [2023-04-29 09:45:08,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:08,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +47: [2023-04-29 09:45:08,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:08,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... +31: [2023-04-29 09:45:08,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:08,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +27: [2023-04-29 09:45:08,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +45: [2023-04-29 09:45:08,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +16: [2023-04-29 09:45:08,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:08,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:08,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:08,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:08,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +34: [2023-04-29 09:45:08,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +12: [2023-04-29 09:45:08,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +12: [2023-04-29 09:45:08,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +12: [2023-04-29 09:45:08,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +12: [2023-04-29 09:45:08,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +34: [2023-04-29 09:45:08,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +27: [2023-04-29 09:45:08,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:08,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:08,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +27: [2023-04-29 09:45:08,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +45: [2023-04-29 09:45:08,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +45: [2023-04-29 09:45:08,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_00-model_states.pt. +37: [2023-04-29 09:45:08,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +37: [2023-04-29 09:45:08,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +37: [2023-04-29 09:45:08,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +45: [2023-04-29 09:45:08,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +44: [2023-04-29 09:45:08,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:08,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... +19: [2023-04-29 09:45:08,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +34: [2023-04-29 09:45:08,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +34: [2023-04-29 09:45:08,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +34: [2023-04-29 09:45:08,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +44: [2023-04-29 09:45:08,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +34: [2023-04-29 09:45:08,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +44: [2023-04-29 09:45:08,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +24: [2023-04-29 09:45:08,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +45: [2023-04-29 09:45:08,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:08,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +48: [2023-04-29 09:45:08,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +19: [2023-04-29 09:45:08,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:08,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:08,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:08,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +22: [2023-04-29 09:45:08,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +22: [2023-04-29 09:45:08,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +22: [2023-04-29 09:45:08,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +20: [2023-04-29 09:45:08,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +37: [2023-04-29 09:45:08,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:08,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +54: [2023-04-29 09:45:08,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +54: [2023-04-29 09:45:08,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +12: [2023-04-29 09:45:08,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:08,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +12: [2023-04-29 09:45:08,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:08,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +12: [2023-04-29 09:45:08,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:08,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +37: [2023-04-29 09:45:08,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:08,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:08,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:08,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +20: [2023-04-29 09:45:08,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:08,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:08,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +20: [2023-04-29 09:45:08,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +20: [2023-04-29 09:45:08,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +15: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +15: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +15: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +20: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +15: [2023-04-29 09:45:08,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +22: [2023-04-29 09:45:08,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +24: [2023-04-29 09:45:08,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +22: [2023-04-29 09:45:08,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:08,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +22: [2023-04-29 09:45:08,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +37: [2023-04-29 09:45:08,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +27: [2023-04-29 09:45:08,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +54: [2023-04-29 09:45:08,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +54: [2023-04-29 09:45:08,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +54: [2023-04-29 09:45:08,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +24: [2023-04-29 09:45:08,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +16: [2023-04-29 09:45:08,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +54: [2023-04-29 09:45:08,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +24: [2023-04-29 09:45:08,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:08,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +27: [2023-04-29 09:45:08,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:08,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:08,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +58: [2023-04-29 09:45:08,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +16: [2023-04-29 09:45:08,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +16: [2023-04-29 09:45:08,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:08,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +27: [2023-04-29 09:45:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +27: [2023-04-29 09:45:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +19: [2023-04-29 09:45:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +16: [2023-04-29 09:45:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +58: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +37: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +34: [2023-04-29 09:45:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +24: [2023-04-29 09:45:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +16: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +20: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +34: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +19: [2023-04-29 09:45:08,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +24: [2023-04-29 09:45:08,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:08,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +20: [2023-04-29 09:45:08,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +11: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +11: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +11: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +11: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +27: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +14: [2023-04-29 09:45:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +14: [2023-04-29 09:45:08,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:08,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +16: [2023-04-29 09:45:08,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +15: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +15: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +19: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:08,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 0: [2023-04-29 09:45:08,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:08,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:08,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:08,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +16: [2023-04-29 09:45:08,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:08,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +19: [2023-04-29 09:45:08,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:08,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +19: [2023-04-29 09:45:08,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +12: [2023-04-29 09:45:08,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +12: [2023-04-29 09:45:08,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:08,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +12: [2023-04-29 09:45:08,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:08,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:08,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:08,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +12: [2023-04-29 09:45:08,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +12: [2023-04-29 09:45:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +26: [2023-04-29 09:45:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:08,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:08,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +20: [2023-04-29 09:45:08,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +25: [2023-04-29 09:45:08,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:08,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +25: [2023-04-29 09:45:08,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:08,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:08,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +25: [2023-04-29 09:45:08,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:08,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:08,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +25: [2023-04-29 09:45:08,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +13: [2023-04-29 09:45:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +13: [2023-04-29 09:45:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +13: [2023-04-29 09:45:08,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +24: [2023-04-29 09:45:08,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +37: [2023-04-29 09:45:08,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +24: [2023-04-29 09:45:08,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +22: [2023-04-29 09:45:08,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:08,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:08,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:08,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:08,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +20: [2023-04-29 09:45:08,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +22: [2023-04-29 09:45:08,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +22: [2023-04-29 09:45:08,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +24: [2023-04-29 09:45:08,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +37: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +22: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +39: [2023-04-29 09:45:08,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +32: [2023-04-29 09:45:08,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +53: [2023-04-29 09:45:08,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +53: [2023-04-29 09:45:08,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +29: [2023-04-29 09:45:08,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +53: [2023-04-29 09:45:08,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +53: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +29: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +29: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +37: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +29: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +39: [2023-04-29 09:45:08,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +32: [2023-04-29 09:45:08,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:08,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +22: [2023-04-29 09:45:08,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +53: [2023-04-29 09:45:08,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +43: [2023-04-29 09:45:08,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:08,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +13: [2023-04-29 09:45:08,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:08,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +13: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +16: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +13: [2023-04-29 09:45:08,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +10: [2023-04-29 09:45:08,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +10: [2023-04-29 09:45:08,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +10: [2023-04-29 09:45:08,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:08,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:08,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:08,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +29: [2023-04-29 09:45:08,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 0: [2023-04-29 09:45:08,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +29: [2023-04-29 09:45:08,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:08,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:08,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:08,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:08,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +15: [2023-04-29 09:45:08,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 5: [2023-04-29 09:45:08,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 5: [2023-04-29 09:45:08,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +63: [2023-04-29 09:45:08,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +63: [2023-04-29 09:45:08,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +63: [2023-04-29 09:45:08,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +63: [2023-04-29 09:45:08,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +28: [2023-04-29 09:45:08,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +28: [2023-04-29 09:45:08,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +28: [2023-04-29 09:45:08,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:08,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +15: [2023-04-29 09:45:08,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +26: [2023-04-29 09:45:08,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +16: [2023-04-29 09:45:08,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:08,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:08,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:08,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 9: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 9: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +51: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +51: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +52: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +14: [2023-04-29 09:45:08,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +26: [2023-04-29 09:45:08,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +51: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +51: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +21: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +21: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +21: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +52: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +21: [2023-04-29 09:45:08,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +25: [2023-04-29 09:45:08,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +51: [2023-04-29 09:45:08,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:08,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:08,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +26: [2023-04-29 09:45:08,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +51: [2023-04-29 09:45:08,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +10: [2023-04-29 09:45:08,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:08,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:08,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +14: [2023-04-29 09:45:08,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +63: [2023-04-29 09:45:08,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +63: [2023-04-29 09:45:08,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +63: [2023-04-29 09:45:08,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +14: [2023-04-29 09:45:08,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +54: [2023-04-29 09:45:08,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +25: [2023-04-29 09:45:08,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +54: [2023-04-29 09:45:08,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +26: [2023-04-29 09:45:08,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +39: [2023-04-29 09:45:08,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:08,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 7: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 7: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 7: [2023-04-29 09:45:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +43: [2023-04-29 09:45:08,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +26: [2023-04-29 09:45:08,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +39: [2023-04-29 09:45:08,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +26: [2023-04-29 09:45:08,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:08,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:08,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +18: [2023-04-29 09:45:08,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +25: [2023-04-29 09:45:08,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:08,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +43: [2023-04-29 09:45:08,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +54: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +39: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +54: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +18: [2023-04-29 09:45:08,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +52: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +21: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +39: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +10: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +10: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +18: [2023-04-29 09:45:08,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +52: [2023-04-29 09:45:08,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:08,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +53: [2023-04-29 09:45:08,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +53: [2023-04-29 09:45:08,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +54: [2023-04-29 09:45:08,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:08,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +54: [2023-04-29 09:45:08,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:08,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:08,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +53: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +23: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +23: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +23: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +23: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +13: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +38: [2023-04-29 09:45:08,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +38: [2023-04-29 09:45:08,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +10: [2023-04-29 09:45:08,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:08,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +38: [2023-04-29 09:45:08,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +26: [2023-04-29 09:45:08,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:08,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +13: [2023-04-29 09:45:08,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +25: [2023-04-29 09:45:08,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +25: [2023-04-29 09:45:08,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:08,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +39: [2023-04-29 09:45:08,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +34: [2023-04-29 09:45:08,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +34: [2023-04-29 09:45:08,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +34: [2023-04-29 09:45:08,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +32: [2023-04-29 09:45:08,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +30: [2023-04-29 09:45:08,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +30: [2023-04-29 09:45:08,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +39: [2023-04-29 09:45:08,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +43: [2023-04-29 09:45:08,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:08,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:08,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +43: [2023-04-29 09:45:08,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +39: [2023-04-29 09:45:08,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +39: [2023-04-29 09:45:08,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:08,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +53: [2023-04-29 09:45:08,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:08,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:08,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +53: [2023-04-29 09:45:08,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +53: [2023-04-29 09:45:08,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:08,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +29: [2023-04-29 09:45:08,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +29: [2023-04-29 09:45:08,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +28: [2023-04-29 09:45:08,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +38: [2023-04-29 09:45:08,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:08,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +35: [2023-04-29 09:45:08,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +35: [2023-04-29 09:45:08,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +53: [2023-04-29 09:45:08,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:08,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +23: [2023-04-29 09:45:08,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +10: [2023-04-29 09:45:08,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +38: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +23: [2023-04-29 09:45:08,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +60: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +14: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +36: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +36: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +36: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +60: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +23: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +14: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +60: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +35: [2023-04-29 09:45:08,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +35: [2023-04-29 09:45:08,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:08,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:08,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +28: [2023-04-29 09:45:08,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +51: [2023-04-29 09:45:08,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +51: [2023-04-29 09:45:08,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +10: [2023-04-29 09:45:08,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +60: [2023-04-29 09:45:08,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:08,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +41: [2023-04-29 09:45:08,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +41: [2023-04-29 09:45:08,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +41: [2023-04-29 09:45:08,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +60: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +60: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +25: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +25: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +25: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +60: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +25: [2023-04-29 09:45:08,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +28: [2023-04-29 09:45:08,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 9: [2023-04-29 09:45:08,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +10: [2023-04-29 09:45:08,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +51: [2023-04-29 09:45:08,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +51: [2023-04-29 09:45:08,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +41: [2023-04-29 09:45:08,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:08,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:08,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +41: [2023-04-29 09:45:08,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +34: [2023-04-29 09:45:08,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +34: [2023-04-29 09:45:08,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +34: [2023-04-29 09:45:08,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +26: [2023-04-29 09:45:08,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +26: [2023-04-29 09:45:08,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +26: [2023-04-29 09:45:08,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 5: [2023-04-29 09:45:08,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +26: [2023-04-29 09:45:08,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +30: [2023-04-29 09:45:08,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +30: [2023-04-29 09:45:08,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:08,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +30: [2023-04-29 09:45:08,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +34: [2023-04-29 09:45:08,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +30: [2023-04-29 09:45:08,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +28: [2023-04-29 09:45:08,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +36: [2023-04-29 09:45:08,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +10: [2023-04-29 09:45:08,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +36: [2023-04-29 09:45:08,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +36: [2023-04-29 09:45:08,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +14: [2023-04-29 09:45:08,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +36: [2023-04-29 09:45:08,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +36: [2023-04-29 09:45:08,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +36: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +10: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +36: [2023-04-29 09:45:08,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +10: [2023-04-29 09:45:08,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +25: [2023-04-29 09:45:08,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +51: [2023-04-29 09:45:08,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +32: [2023-04-29 09:45:08,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +14: [2023-04-29 09:45:08,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +51: [2023-04-29 09:45:08,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +32: [2023-04-29 09:45:08,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +54: [2023-04-29 09:45:08,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +26: [2023-04-29 09:45:08,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:08,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +17: [2023-04-29 09:45:08,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +10: [2023-04-29 09:45:08,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +25: [2023-04-29 09:45:08,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +54: [2023-04-29 09:45:08,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +25: [2023-04-29 09:45:08,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:08,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +17: [2023-04-29 09:45:08,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +26: [2023-04-29 09:45:08,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:08,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +26: [2023-04-29 09:45:08,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +26: [2023-04-29 09:45:08,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +63: [2023-04-29 09:45:08,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +31: [2023-04-29 09:45:08,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +31: [2023-04-29 09:45:08,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +31: [2023-04-29 09:45:08,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_07-model_01-model_states.pt. +28: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +63: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +18: [2023-04-29 09:45:08,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +28: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +63: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +63: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:08,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +18: [2023-04-29 09:45:08,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +63: [2023-04-29 09:45:08,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +18: [2023-04-29 09:45:08,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +62: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +62: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +62: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +58: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +58: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +58: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +62: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +58: [2023-04-29 09:45:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +18: [2023-04-29 09:45:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +35: [2023-04-29 09:45:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:08,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +22: [2023-04-29 09:45:08,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +22: [2023-04-29 09:45:08,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:08,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +32: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +32: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +18: [2023-04-29 09:45:08,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +32: [2023-04-29 09:45:08,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +60: [2023-04-29 09:45:08,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +17: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +17: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +33: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +29: [2023-04-29 09:45:08,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +32: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:08,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +29: [2023-04-29 09:45:08,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +29: [2023-04-29 09:45:08,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +22: [2023-04-29 09:45:08,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:08,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +18: [2023-04-29 09:45:08,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +60: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +17: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +18: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +29: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +35: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +35: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +21: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +60: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +60: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +41: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +41: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +33: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:08,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:08,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +54: [2023-04-29 09:45:08,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +54: [2023-04-29 09:45:08,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +31: [2023-04-29 09:45:08,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +54: [2023-04-29 09:45:08,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +31: [2023-04-29 09:45:08,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +34: [2023-04-29 09:45:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +30: [2023-04-29 09:45:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +41: [2023-04-29 09:45:08,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +31: [2023-04-29 09:45:08,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt... +21: [2023-04-29 09:45:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:08,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:08,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +21: [2023-04-29 09:45:08,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:08,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:08,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +34: [2023-04-29 09:45:08,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +30: [2023-04-29 09:45:08,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:08,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +41: [2023-04-29 09:45:08,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +30: [2023-04-29 09:45:08,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +34: [2023-04-29 09:45:08,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +34: [2023-04-29 09:45:08,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +35: [2023-04-29 09:45:08,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:08,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +30: [2023-04-29 09:45:08,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +30: [2023-04-29 09:45:08,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +58: [2023-04-29 09:45:08,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:08,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:08,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +14: [2023-04-29 09:45:08,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:08,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +34: [2023-04-29 09:45:08,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +14: [2023-04-29 09:45:08,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +35: [2023-04-29 09:45:08,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +62: [2023-04-29 09:45:08,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +60: [2023-04-29 09:45:08,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:08,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +35: [2023-04-29 09:45:08,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +49: [2023-04-29 09:45:08,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +49: [2023-04-29 09:45:08,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +35: [2023-04-29 09:45:08,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +41: [2023-04-29 09:45:08,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +26: [2023-04-29 09:45:08,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +41: [2023-04-29 09:45:08,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:08,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +25: [2023-04-29 09:45:08,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +60: [2023-04-29 09:45:08,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +41: [2023-04-29 09:45:08,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +30: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +30: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +30: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +30: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +38: [2023-04-29 09:45:08,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +38: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +38: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +38: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +38: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +38: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +57: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +57: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +57: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +36: [2023-04-29 09:45:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +59: [2023-04-29 09:45:08,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +43: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +43: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +43: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +43: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +36: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +41: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +26: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +59: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +36: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 6: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 6: [2023-04-29 09:45:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +35: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +36: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +35: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +35: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +30: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +36: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +14: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +56: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +56: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +30: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +56: [2023-04-29 09:45:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +56: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +38: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +38: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +35: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +38: [2023-04-29 09:45:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:08,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +57: [2023-04-29 09:45:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +25: [2023-04-29 09:45:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +36: [2023-04-29 09:45:08,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +57: [2023-04-29 09:45:08,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +26: [2023-04-29 09:45:08,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +36: [2023-04-29 09:45:08,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +26: [2023-04-29 09:45:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +57: [2023-04-29 09:45:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +57: [2023-04-29 09:45:08,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +63: [2023-04-29 09:45:08,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +49: [2023-04-29 09:45:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +57: [2023-04-29 09:45:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +25: [2023-04-29 09:45:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +25: [2023-04-29 09:45:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +63: [2023-04-29 09:45:08,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +26: [2023-04-29 09:45:08,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +25: [2023-04-29 09:45:08,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +11: [2023-04-29 09:45:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +11: [2023-04-29 09:45:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +28: [2023-04-29 09:45:08,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +31: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +11: [2023-04-29 09:45:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +49: [2023-04-29 09:45:08,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:08,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +42: [2023-04-29 09:45:08,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +42: [2023-04-29 09:45:08,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +49: [2023-04-29 09:45:08,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:08,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +32: [2023-04-29 09:45:08,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +56: [2023-04-29 09:45:08,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +56: [2023-04-29 09:45:08,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +11: [2023-04-29 09:45:08,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +17: [2023-04-29 09:45:08,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +56: [2023-04-29 09:45:08,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +56: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +11: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +29: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +21: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +57: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +11: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +50: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +50: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +32: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +59: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +33: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +57: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +50: [2023-04-29 09:45:08,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +50: [2023-04-29 09:45:08,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +59: [2023-04-29 09:45:08,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +32: [2023-04-29 09:45:08,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +36: [2023-04-29 09:45:08,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +35: [2023-04-29 09:45:08,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +36: [2023-04-29 09:45:08,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +28: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +43: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +35: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:08,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +43: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +32: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +22: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +56: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +29: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +17: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +56: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +56: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +48: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +48: [2023-04-29 09:45:08,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +61: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +33: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +22: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +61: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +56: [2023-04-29 09:45:08,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +33: [2023-04-29 09:45:08,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 9: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +32: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +22: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +56: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +63: [2023-04-29 09:45:08,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +20: [2023-04-29 09:45:08,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +20: [2023-04-29 09:45:08,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +32: [2023-04-29 09:45:08,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +35: [2023-04-29 09:45:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +61: [2023-04-29 09:45:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +20: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +37: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +61: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +28: [2023-04-29 09:45:08,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +28: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 9: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +31: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_00-model_states.pt. +63: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +21: [2023-04-29 09:45:08,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +61: [2023-04-29 09:45:08,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +63: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +58: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +63: [2023-04-29 09:45:08,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +15: [2023-04-29 09:45:08,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +42: [2023-04-29 09:45:08,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt... +42: [2023-04-29 09:45:08,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +42: [2023-04-29 09:45:08,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +29: [2023-04-29 09:45:08,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +48: [2023-04-29 09:45:08,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +48: [2023-04-29 09:45:08,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +48: [2023-04-29 09:45:08,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +23: [2023-04-29 09:45:08,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +58: [2023-04-29 09:45:08,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +58: [2023-04-29 09:45:08,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +15: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +22: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +15: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +50: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +23: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +23: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +42: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +50: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +23: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +23: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +58: [2023-04-29 09:45:08,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +23: [2023-04-29 09:45:08,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:08,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:08,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +15: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +42: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +27: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +27: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +27: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +42: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +50: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:08,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +33: [2023-04-29 09:45:08,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +58: [2023-04-29 09:45:08,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:08,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +44: [2023-04-29 09:45:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +44: [2023-04-29 09:45:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +48: [2023-04-29 09:45:08,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +53: [2023-04-29 09:45:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +53: [2023-04-29 09:45:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +53: [2023-04-29 09:45:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +53: [2023-04-29 09:45:08,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +49: [2023-04-29 09:45:08,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +45: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 0: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +52: [2023-04-29 09:45:08,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +22: [2023-04-29 09:45:08,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:08,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +50: [2023-04-29 09:45:08,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +20: [2023-04-29 09:45:08,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:08,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +22: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +33: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +48: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +29: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +49: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +22: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +45: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +45: [2023-04-29 09:45:08,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +49: [2023-04-29 09:45:08,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +55: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +55: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +55: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +55: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +61: [2023-04-29 09:45:08,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +29: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +52: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +61: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +30: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +30: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +20: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +52: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +52: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +20: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +61: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +52: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +52: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +12: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +12: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +52: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +49: [2023-04-29 09:45:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +33: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:08,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +12: [2023-04-29 09:45:08,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:08,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +21: [2023-04-29 09:45:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +20: [2023-04-29 09:45:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +39: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +39: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +39: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 7: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +39: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +30: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +30: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +37: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +37: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +37: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +12: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:08,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +12: [2023-04-29 09:45:08,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:08,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +59: [2023-04-29 09:45:08,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +12: [2023-04-29 09:45:08,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +59: [2023-04-29 09:45:08,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +27: [2023-04-29 09:45:08,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +57: [2023-04-29 09:45:08,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +54: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +54: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +27: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +59: [2023-04-29 09:45:08,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +49: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +44: [2023-04-29 09:45:08,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:08,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +54: [2023-04-29 09:45:08,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +44: [2023-04-29 09:45:08,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +59: [2023-04-29 09:45:08,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:08,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +40: [2023-04-29 09:45:08,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +40: [2023-04-29 09:45:08,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +53: [2023-04-29 09:45:08,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +40: [2023-04-29 09:45:08,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +27: [2023-04-29 09:45:08,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +40: [2023-04-29 09:45:08,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +27: [2023-04-29 09:45:08,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:08,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +49: [2023-04-29 09:45:08,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +49: [2023-04-29 09:45:08,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +53: [2023-04-29 09:45:08,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +16: [2023-04-29 09:45:08,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +16: [2023-04-29 09:45:08,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +13: [2023-04-29 09:45:08,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +13: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +57: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +13: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +56: [2023-04-29 09:45:08,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 6: [2023-04-29 09:45:08,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +11: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +59: [2023-04-29 09:45:08,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +56: [2023-04-29 09:45:08,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:08,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +13: [2023-04-29 09:45:08,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +61: [2023-04-29 09:45:08,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:08,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +47: [2023-04-29 09:45:08,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +35: [2023-04-29 09:45:08,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +61: [2023-04-29 09:45:08,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +61: [2023-04-29 09:45:08,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +55: [2023-04-29 09:45:08,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:08,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +55: [2023-04-29 09:45:08,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +38: [2023-04-29 09:45:08,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +18: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +18: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +37: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +57: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +57: [2023-04-29 09:45:08,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +37: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +59: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +24: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +39: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +24: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +56: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +56: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +30: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +47: [2023-04-29 09:45:08,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +58: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +58: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +58: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +58: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +35: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:08,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +30: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +39: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +17: [2023-04-29 09:45:08,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +24: [2023-04-29 09:45:08,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +40: [2023-04-29 09:45:08,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +18: [2023-04-29 09:45:08,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +60: [2023-04-29 09:45:08,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +60: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +57: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +17: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +60: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +56: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +56: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +60: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +35: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +54: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +58: [2023-04-29 09:45:08,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +58: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +56: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +59: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +24: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +20: [2023-04-29 09:45:08,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +62: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:08,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:08,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +56: [2023-04-29 09:45:08,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +35: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +40: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +40: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +42: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +54: [2023-04-29 09:45:08,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +40: [2023-04-29 09:45:08,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +43: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +61: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +40: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +62: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +43: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +40: [2023-04-29 09:45:08,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +20: [2023-04-29 09:45:08,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +54: [2023-04-29 09:45:08,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +15: [2023-04-29 09:45:08,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +54: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +23: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +11: [2023-04-29 09:45:08,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +56: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +11: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +42: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +50: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +59: [2023-04-29 09:45:08,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +20: [2023-04-29 09:45:08,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +20: [2023-04-29 09:45:08,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +23: [2023-04-29 09:45:08,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +62: [2023-04-29 09:45:08,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +62: [2023-04-29 09:45:08,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +56: [2023-04-29 09:45:08,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +16: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +37: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +62: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +56: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +48: [2023-04-29 09:45:08,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +15: [2023-04-29 09:45:08,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +62: [2023-04-29 09:45:08,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +34: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +34: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:08,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +57: [2023-04-29 09:45:08,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +16: [2023-04-29 09:45:08,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +50: [2023-04-29 09:45:08,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +16: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +16: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +15: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +11: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +34: [2023-04-29 09:45:08,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +34: [2023-04-29 09:45:08,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +42: [2023-04-29 09:45:08,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +43: [2023-04-29 09:45:08,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +43: [2023-04-29 09:45:08,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +20: [2023-04-29 09:45:08,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +43: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +49: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +48: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +48: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +34: [2023-04-29 09:45:08,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +45: [2023-04-29 09:45:08,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +34: [2023-04-29 09:45:08,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +19: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +19: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +53: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +34: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +19: [2023-04-29 09:45:08,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +20: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +47: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +47: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +47: [2023-04-29 09:45:08,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +47: [2023-04-29 09:45:08,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +16: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +42: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +43: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +43: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +43: [2023-04-29 09:45:08,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +16: [2023-04-29 09:45:08,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +42: [2023-04-29 09:45:08,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +60: [2023-04-29 09:45:08,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +60: [2023-04-29 09:45:08,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +43: [2023-04-29 09:45:08,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +52: [2023-04-29 09:45:08,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +52: [2023-04-29 09:45:08,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +12: [2023-04-29 09:45:08,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +61: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +53: [2023-04-29 09:45:08,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +53: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +53: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +53: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +19: [2023-04-29 09:45:08,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +20: [2023-04-29 09:45:08,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +19: [2023-04-29 09:45:08,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +48: [2023-04-29 09:45:08,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +48: [2023-04-29 09:45:08,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +48: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +60: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +56: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +52: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +33: [2023-04-29 09:45:08,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +42: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:08,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +15: [2023-04-29 09:45:08,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:08,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:08,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:08,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +53: [2023-04-29 09:45:08,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +12: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +12: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +23: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:08,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +33: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +55: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +48: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +55: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +55: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +49: [2023-04-29 09:45:08,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +23: [2023-04-29 09:45:08,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +23: [2023-04-29 09:45:08,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +19: [2023-04-29 09:45:08,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +42: [2023-04-29 09:45:08,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +31: [2023-04-29 09:45:08,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +31: [2023-04-29 09:45:08,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +37: [2023-04-29 09:45:08,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +49: [2023-04-29 09:45:08,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +48: [2023-04-29 09:45:08,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +15: [2023-04-29 09:45:08,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +59: [2023-04-29 09:45:08,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +59: [2023-04-29 09:45:08,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +31: [2023-04-29 09:45:08,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +61: [2023-04-29 09:45:08,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +61: [2023-04-29 09:45:08,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +46: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +37: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +37: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +15: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:08,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +46: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +46: [2023-04-29 09:45:08,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +50: [2023-04-29 09:45:08,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +50: [2023-04-29 09:45:08,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +51: [2023-04-29 09:45:08,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +51: [2023-04-29 09:45:08,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +48: [2023-04-29 09:45:08,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +48: [2023-04-29 09:45:08,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +31: [2023-04-29 09:45:08,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +31: [2023-04-29 09:45:08,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +53: [2023-04-29 09:45:08,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +50: [2023-04-29 09:45:08,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +45: [2023-04-29 09:45:08,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +48: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +10: [2023-04-29 09:45:08,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +10: [2023-04-29 09:45:08,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +31: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +42: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +41: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +41: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +10: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:08,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +13: [2023-04-29 09:45:08,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +13: [2023-04-29 09:45:08,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +49: [2023-04-29 09:45:08,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +20: [2023-04-29 09:45:08,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +20: [2023-04-29 09:45:08,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:08,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +51: [2023-04-29 09:45:08,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +51: [2023-04-29 09:45:08,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +20: [2023-04-29 09:45:08,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +45: [2023-04-29 09:45:08,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:08,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +53: [2023-04-29 09:45:08,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +12: [2023-04-29 09:45:08,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:08,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +10: [2023-04-29 09:45:08,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +41: [2023-04-29 09:45:08,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +27: [2023-04-29 09:45:08,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +41: [2023-04-29 09:45:08,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +27: [2023-04-29 09:45:08,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +52: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +27: [2023-04-29 09:45:08,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 0: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +52: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +61: [2023-04-29 09:45:08,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +37: [2023-04-29 09:45:08,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +24: [2023-04-29 09:45:08,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +48: [2023-04-29 09:45:08,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +12: [2023-04-29 09:45:08,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +47: [2023-04-29 09:45:08,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:08,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:08,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +20: [2023-04-29 09:45:08,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +20: [2023-04-29 09:45:08,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +33: [2023-04-29 09:45:08,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +27: [2023-04-29 09:45:08,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +33: [2023-04-29 09:45:08,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +53: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +55: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +27: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +27: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +27: [2023-04-29 09:45:08,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +27: [2023-04-29 09:45:08,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +33: [2023-04-29 09:45:08,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +12: [2023-04-29 09:45:08,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:08,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +37: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +20: [2023-04-29 09:45:08,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +27: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +61: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +14: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +27: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +14: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +63: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +39: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +55: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +63: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +50: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +63: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +14: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:08,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +40: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +18: [2023-04-29 09:45:08,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +37: [2023-04-29 09:45:08,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +18: [2023-04-29 09:45:08,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:08,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +20: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +49: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:08,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +39: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +39: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +33: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +40: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +59: [2023-04-29 09:45:08,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +59: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +55: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +14: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +55: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +39: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +14: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:08,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +40: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +39: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +39: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +39: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +46: [2023-04-29 09:45:08,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +39: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +61: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +61: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +58: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +17: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +58: [2023-04-29 09:45:08,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +12: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +12: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +12: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +12: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +16: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +41: [2023-04-29 09:45:08,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +51: [2023-04-29 09:45:08,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +13: [2023-04-29 09:45:08,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +40: [2023-04-29 09:45:08,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +13: [2023-04-29 09:45:08,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +37: [2023-04-29 09:45:08,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +13: [2023-04-29 09:45:08,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +34: [2023-04-29 09:45:08,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +51: [2023-04-29 09:45:08,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +44: [2023-04-29 09:45:08,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +13: [2023-04-29 09:45:08,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:08,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +18: [2023-04-29 09:45:08,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:08,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +44: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +16: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +16: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +59: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:08,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +41: [2023-04-29 09:45:08,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +59: [2023-04-29 09:45:08,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +45: [2023-04-29 09:45:08,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +51: [2023-04-29 09:45:08,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +40: [2023-04-29 09:45:08,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +24: [2023-04-29 09:45:08,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:08,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +45: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_29-model_01-model_states.pt. +49: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +47: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +44: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +44: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +44: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +44: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +44: [2023-04-29 09:45:08,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +17: [2023-04-29 09:45:08,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +37: [2023-04-29 09:45:08,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +51: [2023-04-29 09:45:08,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +62: [2023-04-29 09:45:08,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +58: [2023-04-29 09:45:08,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:08,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +32: [2023-04-29 09:45:08,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +32: [2023-04-29 09:45:08,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +32: [2023-04-29 09:45:08,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +22: [2023-04-29 09:45:08,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +22: [2023-04-29 09:45:08,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +22: [2023-04-29 09:45:08,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +22: [2023-04-29 09:45:08,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +58: [2023-04-29 09:45:08,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:08,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +43: [2023-04-29 09:45:08,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +32: [2023-04-29 09:45:08,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +62: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +16: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +24: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +32: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +19: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +19: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +63: [2023-04-29 09:45:08,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +34: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +19: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +63: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +19: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +16: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:08,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +20: [2023-04-29 09:45:08,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +63: [2023-04-29 09:45:08,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +34: [2023-04-29 09:45:08,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +43: [2023-04-29 09:45:08,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +53: [2023-04-29 09:45:08,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +18: [2023-04-29 09:45:08,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:08,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +63: [2023-04-29 09:45:08,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +60: [2023-04-29 09:45:08,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +34: [2023-04-29 09:45:08,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +17: [2023-04-29 09:45:08,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +47: [2023-04-29 09:45:08,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +34: [2023-04-29 09:45:08,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +53: [2023-04-29 09:45:08,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +43: [2023-04-29 09:45:08,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +17: [2023-04-29 09:45:08,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +16: [2023-04-29 09:45:08,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +16: [2023-04-29 09:45:08,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +12: [2023-04-29 09:45:08,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +12: [2023-04-29 09:45:08,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +46: [2023-04-29 09:45:08,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +40: [2023-04-29 09:45:08,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +12: [2023-04-29 09:45:08,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +24: [2023-04-29 09:45:08,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +17: [2023-04-29 09:45:08,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +60: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +58: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:08,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +12: [2023-04-29 09:45:08,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +62: [2023-04-29 09:45:08,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +20: [2023-04-29 09:45:08,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:08,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +24: [2023-04-29 09:45:08,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:08,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +34: [2023-04-29 09:45:08,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:08,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +16: [2023-04-29 09:45:08,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:08,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +55: [2023-04-29 09:45:08,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:08,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +20: [2023-04-29 09:45:08,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +60: [2023-04-29 09:45:08,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +60: [2023-04-29 09:45:08,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +16: [2023-04-29 09:45:08,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +45: [2023-04-29 09:45:08,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +34: [2023-04-29 09:45:08,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +47: [2023-04-29 09:45:08,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +62: [2023-04-29 09:45:08,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt... +31: [2023-04-29 09:45:08,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +31: [2023-04-29 09:45:08,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +43: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +22: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +19: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:08,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +16: [2023-04-29 09:45:08,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:08,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +34: [2023-04-29 09:45:08,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +10: [2023-04-29 09:45:08,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +31: [2023-04-29 09:45:08,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +62: [2023-04-29 09:45:08,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +22: [2023-04-29 09:45:08,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +43: [2023-04-29 09:45:08,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:08,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:08,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +34: [2023-04-29 09:45:08,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +19: [2023-04-29 09:45:08,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:08,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:08,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:08,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +20: [2023-04-29 09:45:08,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +19: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +10: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +10: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +10: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +10: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +10: [2023-04-29 09:45:08,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +10: [2023-04-29 09:45:08,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +39: [2023-04-29 09:45:08,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +43: [2023-04-29 09:45:08,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:08,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +27: [2023-04-29 09:45:08,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +27: [2023-04-29 09:45:08,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +34: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +51: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +14: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:08,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:08,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:08,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +51: [2023-04-29 09:45:08,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +24: [2023-04-29 09:45:08,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +53: [2023-04-29 09:45:08,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +24: [2023-04-29 09:45:08,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +39: [2023-04-29 09:45:08,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:08,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +33: [2023-04-29 09:45:08,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +19: [2023-04-29 09:45:08,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +36: [2023-04-29 09:45:08,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +36: [2023-04-29 09:45:08,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +36: [2023-04-29 09:45:08,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +51: [2023-04-29 09:45:08,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +39: [2023-04-29 09:45:08,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:08,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +51: [2023-04-29 09:45:08,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +51: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +53: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +31: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +34: [2023-04-29 09:45:08,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:08,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +16: [2023-04-29 09:45:08,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:08,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +32: [2023-04-29 09:45:08,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +32: [2023-04-29 09:45:08,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +20: [2023-04-29 09:45:08,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:08,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +33: [2023-04-29 09:45:08,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +55: [2023-04-29 09:45:08,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +14: [2023-04-29 09:45:08,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +14: [2023-04-29 09:45:08,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +16: [2023-04-29 09:45:08,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:08,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:08,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:08,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +33: [2023-04-29 09:45:08,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +20: [2023-04-29 09:45:08,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +47: [2023-04-29 09:45:08,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +31: [2023-04-29 09:45:08,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +44: [2023-04-29 09:45:08,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +10: [2023-04-29 09:45:08,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:08,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +25: [2023-04-29 09:45:08,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +25: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +25: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +41: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +25: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +41: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +41: [2023-04-29 09:45:08,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +15: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +28: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +15: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +44: [2023-04-29 09:45:08,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +47: [2023-04-29 09:45:08,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +47: [2023-04-29 09:45:08,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +28: [2023-04-29 09:45:08,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +44: [2023-04-29 09:45:08,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +28: [2023-04-29 09:45:08,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +41: [2023-04-29 09:45:08,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +41: [2023-04-29 09:45:08,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +10: [2023-04-29 09:45:08,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +41: [2023-04-29 09:45:08,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +41: [2023-04-29 09:45:08,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +41: [2023-04-29 09:45:08,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +54: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +46: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:08,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +41: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +27: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +39: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +28: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +41: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:08,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:08,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +51: [2023-04-29 09:45:08,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:08,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +54: [2023-04-29 09:45:08,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +54: [2023-04-29 09:45:08,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +54: [2023-04-29 09:45:08,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +10: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +19: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +19: [2023-04-29 09:45:08,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +54: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +54: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +19: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +35: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +12: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +46: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +35: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +35: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +21: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +21: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +27: [2023-04-29 09:45:08,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +41: [2023-04-29 09:45:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +54: [2023-04-29 09:45:08,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +21: [2023-04-29 09:45:08,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +51: [2023-04-29 09:45:08,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +12: [2023-04-29 09:45:08,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +41: [2023-04-29 09:45:08,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +39: [2023-04-29 09:45:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +27: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +32: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +10: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +22: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +12: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +10: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +35: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +38: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +38: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +38: [2023-04-29 09:45:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +54: [2023-04-29 09:45:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +54: [2023-04-29 09:45:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +51: [2023-04-29 09:45:08,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +35: [2023-04-29 09:45:08,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +12: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +32: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +39: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +39: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +10: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +38: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +12: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +54: [2023-04-29 09:45:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +22: [2023-04-29 09:45:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +10: [2023-04-29 09:45:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +54: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +12: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +60: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +51: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +60: [2023-04-29 09:45:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:08,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +19: [2023-04-29 09:45:08,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:08,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +45: [2023-04-29 09:45:08,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +14: [2023-04-29 09:45:08,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +44: [2023-04-29 09:45:08,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +12: [2023-04-29 09:45:08,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +14: [2023-04-29 09:45:08,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:08,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:08,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:08,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +15: [2023-04-29 09:45:08,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +60: [2023-04-29 09:45:08,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +32: [2023-04-29 09:45:08,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +15: [2023-04-29 09:45:08,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:08,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:08,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +25: [2023-04-29 09:45:08,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +51: [2023-04-29 09:45:08,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +22: [2023-04-29 09:45:08,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +25: [2023-04-29 09:45:08,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +58: [2023-04-29 09:45:08,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +58: [2023-04-29 09:45:08,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +58: [2023-04-29 09:45:08,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +45: [2023-04-29 09:45:08,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +45: [2023-04-29 09:45:08,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +58: [2023-04-29 09:45:08,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +24: [2023-04-29 09:45:08,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +28: [2023-04-29 09:45:08,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +28: [2023-04-29 09:45:08,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +34: [2023-04-29 09:45:08,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. +24: [2023-04-29 09:45:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +32: [2023-04-29 09:45:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +22: [2023-04-29 09:45:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +28: [2023-04-29 09:45:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +28: [2023-04-29 09:45:08,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +22: [2023-04-29 09:45:08,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +45: [2023-04-29 09:45:08,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +34: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +25: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +51: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +22: [2023-04-29 09:45:08,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +24: [2023-04-29 09:45:08,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 0: [2023-04-29 09:45:08,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +22: [2023-04-29 09:45:08,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +45: [2023-04-29 09:45:08,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +34: [2023-04-29 09:45:08,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +18: [2023-04-29 09:45:08,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +18: [2023-04-29 09:45:08,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +18: [2023-04-29 09:45:08,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +45: [2023-04-29 09:45:08,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt... +63: [2023-04-29 09:45:08,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +63: [2023-04-29 09:45:08,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +63: [2023-04-29 09:45:08,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +21: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +34: [2023-04-29 09:45:08,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +22: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +63: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +21: [2023-04-29 09:45:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:08,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +19: [2023-04-29 09:45:08,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 5: [2023-04-29 09:45:08,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 5: [2023-04-29 09:45:08,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +63: [2023-04-29 09:45:08,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +63: [2023-04-29 09:45:08,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +29: [2023-04-29 09:45:08,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +29: [2023-04-29 09:45:08,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +34: [2023-04-29 09:45:08,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +29: [2023-04-29 09:45:08,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +19: [2023-04-29 09:45:08,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +19: [2023-04-29 09:45:08,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +24: [2023-04-29 09:45:08,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +52: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +52: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +52: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +52: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +32: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +11: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +11: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +11: [2023-04-29 09:45:08,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +11: [2023-04-29 09:45:08,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +34: [2023-04-29 09:45:08,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +24: [2023-04-29 09:45:08,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +19: [2023-04-29 09:45:08,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +58: [2023-04-29 09:45:08,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +51: [2023-04-29 09:45:08,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +32: [2023-04-29 09:45:08,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:08,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:08,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +58: [2023-04-29 09:45:08,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +58: [2023-04-29 09:45:08,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +34: [2023-04-29 09:45:08,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +24: [2023-04-29 09:45:08,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +51: [2023-04-29 09:45:08,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +30: [2023-04-29 09:45:08,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +30: [2023-04-29 09:45:08,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +24: [2023-04-29 09:45:08,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +36: [2023-04-29 09:45:08,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +30: [2023-04-29 09:45:08,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +30: [2023-04-29 09:45:08,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +32: [2023-04-29 09:45:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:08,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +41: [2023-04-29 09:45:08,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +41: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +13: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +13: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +10: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +13: [2023-04-29 09:45:08,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +36: [2023-04-29 09:45:08,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +41: [2023-04-29 09:45:08,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +26: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +26: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +26: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +36: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +26: [2023-04-29 09:45:08,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 8: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +28: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:08,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +18: [2023-04-29 09:45:08,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +10: [2023-04-29 09:45:08,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +10: [2023-04-29 09:45:08,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +29: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +19: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +35: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +11: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +29: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +29: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:08,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +18: [2023-04-29 09:45:08,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +54: [2023-04-29 09:45:08,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +28: [2023-04-29 09:45:08,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +28: [2023-04-29 09:45:08,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +36: [2023-04-29 09:45:08,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 7: [2023-04-29 09:45:08,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. + 7: [2023-04-29 09:45:08,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +52: [2023-04-29 09:45:08,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +11: [2023-04-29 09:45:08,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +11: [2023-04-29 09:45:08,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +11: [2023-04-29 09:45:08,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:08,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +10: [2023-04-29 09:45:08,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +52: [2023-04-29 09:45:08,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +32: [2023-04-29 09:45:08,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +32: [2023-04-29 09:45:08,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +35: [2023-04-29 09:45:08,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +51: [2023-04-29 09:45:08,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:08,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +43: [2023-04-29 09:45:08,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +43: [2023-04-29 09:45:08,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 9: [2023-04-29 09:45:08,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +51: [2023-04-29 09:45:08,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:08,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +14: [2023-04-29 09:45:08,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +14: [2023-04-29 09:45:08,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +60: [2023-04-29 09:45:08,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +49: [2023-04-29 09:45:08,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +30: [2023-04-29 09:45:08,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +51: [2023-04-29 09:45:08,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +14: [2023-04-29 09:45:08,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +14: [2023-04-29 09:45:08,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +10: [2023-04-29 09:45:08,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +49: [2023-04-29 09:45:08,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +49: [2023-04-29 09:45:08,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +32: [2023-04-29 09:45:08,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +32: [2023-04-29 09:45:08,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +30: [2023-04-29 09:45:08,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +30: [2023-04-29 09:45:08,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +30: [2023-04-29 09:45:08,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +36: [2023-04-29 09:45:08,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:08,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +15: [2023-04-29 09:45:08,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +63: [2023-04-29 09:45:08,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +32: [2023-04-29 09:45:08,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +13: [2023-04-29 09:45:08,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +10: [2023-04-29 09:45:08,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +26: [2023-04-29 09:45:08,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:08,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +15: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +15: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +15: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +41: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +35: [2023-04-29 09:45:08,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +13: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +54: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +33: [2023-04-29 09:45:08,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +38: [2023-04-29 09:45:08,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +38: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +28: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +36: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +17: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +17: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +38: [2023-04-29 09:45:08,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +15: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +63: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +26: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +36: [2023-04-29 09:45:08,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 1: [2023-04-29 09:45:08,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +38: [2023-04-29 09:45:08,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +17: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +17: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +31: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +31: [2023-04-29 09:45:08,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +32: [2023-04-29 09:45:08,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +31: [2023-04-29 09:45:08,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +31: [2023-04-29 09:45:08,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_08-model_01-model_states.pt. +54: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +42: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +54: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +35: [2023-04-29 09:45:08,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:08,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +35: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +35: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 3: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +32: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +60: [2023-04-29 09:45:08,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +28: [2023-04-29 09:45:08,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +59: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +58: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +59: [2023-04-29 09:45:08,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 0: [2023-04-29 09:45:08,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +59: [2023-04-29 09:45:08,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +43: [2023-04-29 09:45:08,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +38: [2023-04-29 09:45:08,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +58: [2023-04-29 09:45:08,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +28: [2023-04-29 09:45:08,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +14: [2023-04-29 09:45:08,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +14: [2023-04-29 09:45:08,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:08,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +49: [2023-04-29 09:45:08,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +49: [2023-04-29 09:45:08,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +38: [2023-04-29 09:45:08,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 0: [2023-04-29 09:45:08,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +14: [2023-04-29 09:45:08,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +58: [2023-04-29 09:45:08,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +38: [2023-04-29 09:45:08,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +60: [2023-04-29 09:45:08,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:08,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:08,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +56: [2023-04-29 09:45:08,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +56: [2023-04-29 09:45:08,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +56: [2023-04-29 09:45:08,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +54: [2023-04-29 09:45:08,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +63: [2023-04-29 09:45:08,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +36: [2023-04-29 09:45:08,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +58: [2023-04-29 09:45:08,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +17: [2023-04-29 09:45:08,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +58: [2023-04-29 09:45:08,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +54: [2023-04-29 09:45:08,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +18: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +56: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +57: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +57: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +57: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +57: [2023-04-29 09:45:08,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +57: [2023-04-29 09:45:08,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +35: [2023-04-29 09:45:08,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +35: [2023-04-29 09:45:08,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +35: [2023-04-29 09:45:08,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +35: [2023-04-29 09:45:08,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +58: [2023-04-29 09:45:08,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +57: [2023-04-29 09:45:08,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +31: [2023-04-29 09:45:08,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +31: [2023-04-29 09:45:08,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +31: [2023-04-29 09:45:08,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +63: [2023-04-29 09:45:08,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +26: [2023-04-29 09:45:08,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +57: [2023-04-29 09:45:08,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +18: [2023-04-29 09:45:08,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +17: [2023-04-29 09:45:08,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +17: [2023-04-29 09:45:08,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +57: [2023-04-29 09:45:08,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +57: [2023-04-29 09:45:08,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +31: [2023-04-29 09:45:08,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt... +18: [2023-04-29 09:45:08,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +18: [2023-04-29 09:45:08,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +36: [2023-04-29 09:45:08,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +26: [2023-04-29 09:45:08,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +26: [2023-04-29 09:45:08,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +26: [2023-04-29 09:45:08,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:08,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +36: [2023-04-29 09:45:08,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +56: [2023-04-29 09:45:08,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +59: [2023-04-29 09:45:08,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +28: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +56: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +59: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +56: [2023-04-29 09:45:08,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +13: [2023-04-29 09:45:08,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +59: [2023-04-29 09:45:08,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +50: [2023-04-29 09:45:08,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +36: [2023-04-29 09:45:08,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +18: [2023-04-29 09:45:08,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +18: [2023-04-29 09:45:08,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +33: [2023-04-29 09:45:08,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +18: [2023-04-29 09:45:08,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +29: [2023-04-29 09:45:08,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +13: [2023-04-29 09:45:08,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +29: [2023-04-29 09:45:08,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +30: [2023-04-29 09:45:08,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +29: [2023-04-29 09:45:08,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +50: [2023-04-29 09:45:08,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +50: [2023-04-29 09:45:08,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +28: [2023-04-29 09:45:08,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +62: [2023-04-29 09:45:08,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +62: [2023-04-29 09:45:08,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +62: [2023-04-29 09:45:08,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +62: [2023-04-29 09:45:08,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +56: [2023-04-29 09:45:08,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +36: [2023-04-29 09:45:08,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:08,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +28: [2023-04-29 09:45:08,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:08,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +30: [2023-04-29 09:45:08,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +50: [2023-04-29 09:45:08,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +50: [2023-04-29 09:45:08,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +38: [2023-04-29 09:45:08,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +33: [2023-04-29 09:45:08,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +13: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +35: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +42: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +42: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +28: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +42: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +30: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +57: [2023-04-29 09:45:08,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +30: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +61: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +61: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +26: [2023-04-29 09:45:08,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +33: [2023-04-29 09:45:08,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:08,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +50: [2023-04-29 09:45:08,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +50: [2023-04-29 09:45:08,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +50: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +57: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +57: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +47: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +13: [2023-04-29 09:45:08,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +13: [2023-04-29 09:45:08,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +13: [2023-04-29 09:45:08,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 8: [2023-04-29 09:45:08,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +61: [2023-04-29 09:45:08,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +47: [2023-04-29 09:45:08,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +47: [2023-04-29 09:45:08,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +26: [2023-04-29 09:45:08,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +38: [2023-04-29 09:45:08,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +61: [2023-04-29 09:45:08,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +38: [2023-04-29 09:45:08,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +13: [2023-04-29 09:45:08,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +28: [2023-04-29 09:45:08,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +47: [2023-04-29 09:45:08,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +13: [2023-04-29 09:45:08,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +36: [2023-04-29 09:45:08,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +28: [2023-04-29 09:45:08,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +28: [2023-04-29 09:45:08,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +47: [2023-04-29 09:45:08,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +53: [2023-04-29 09:45:08,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +53: [2023-04-29 09:45:08,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +53: [2023-04-29 09:45:08,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +14: [2023-04-29 09:45:08,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +36: [2023-04-29 09:45:08,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 8: [2023-04-29 09:45:08,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 8: [2023-04-29 09:45:08,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +55: [2023-04-29 09:45:08,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +55: [2023-04-29 09:45:08,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +55: [2023-04-29 09:45:08,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +55: [2023-04-29 09:45:08,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +47: [2023-04-29 09:45:08,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +26: [2023-04-29 09:45:08,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +43: [2023-04-29 09:45:08,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:08,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +28: [2023-04-29 09:45:08,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +28: [2023-04-29 09:45:08,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +26: [2023-04-29 09:45:08,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +14: [2023-04-29 09:45:08,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +36: [2023-04-29 09:45:08,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:08,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +38: [2023-04-29 09:45:08,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:08,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +26: [2023-04-29 09:45:08,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +26: [2023-04-29 09:45:08,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +49: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +23: [2023-04-29 09:45:08,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +23: [2023-04-29 09:45:08,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +23: [2023-04-29 09:45:08,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +23: [2023-04-29 09:45:08,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +62: [2023-04-29 09:45:08,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +43: [2023-04-29 09:45:08,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +43: [2023-04-29 09:45:08,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:08,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +33: [2023-04-29 09:45:08,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:08,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:08,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +59: [2023-04-29 09:45:08,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +62: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +62: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +42: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +17: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +42: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +21: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +21: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +14: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +42: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +42: [2023-04-29 09:45:08,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:08,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:08,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +17: [2023-04-29 09:45:08,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +26: [2023-04-29 09:45:08,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +49: [2023-04-29 09:45:08,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +50: [2023-04-29 09:45:08,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:08,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +43: [2023-04-29 09:45:08,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:08,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +59: [2023-04-29 09:45:08,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +59: [2023-04-29 09:45:08,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +14: [2023-04-29 09:45:08,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +57: [2023-04-29 09:45:08,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +38: [2023-04-29 09:45:08,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:08,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +21: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +49: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +31: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +31: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +49: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +17: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +56: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +21: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +21: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +21: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +14: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:08,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:08,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +38: [2023-04-29 09:45:08,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:08,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +29: [2023-04-29 09:45:08,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:08,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +29: [2023-04-29 09:45:08,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +53: [2023-04-29 09:45:08,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +38: [2023-04-29 09:45:08,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +42: [2023-04-29 09:45:08,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +31: [2023-04-29 09:45:08,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +59: [2023-04-29 09:45:08,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +31: [2023-04-29 09:45:08,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +44: [2023-04-29 09:45:08,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +44: [2023-04-29 09:45:08,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +31: [2023-04-29 09:45:08,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +55: [2023-04-29 09:45:08,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +14: [2023-04-29 09:45:08,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +31: [2023-04-29 09:45:08,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +55: [2023-04-29 09:45:08,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +17: [2023-04-29 09:45:08,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_00-model_states.pt. +44: [2023-04-29 09:45:08,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 2: [2023-04-29 09:45:08,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +44: [2023-04-29 09:45:08,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +31: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +23: [2023-04-29 09:45:08,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +52: [2023-04-29 09:45:08,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +56: [2023-04-29 09:45:08,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +31: [2023-04-29 09:45:08,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +59: [2023-04-29 09:45:08,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +56: [2023-04-29 09:45:08,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +61: [2023-04-29 09:45:08,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +22: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +22: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +22: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +23: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +22: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +23: [2023-04-29 09:45:08,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:08,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +61: [2023-04-29 09:45:08,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +26: [2023-04-29 09:45:08,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:08,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +57: [2023-04-29 09:45:08,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +17: [2023-04-29 09:45:08,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt... +52: [2023-04-29 09:45:08,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +52: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +52: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +52: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +59: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +59: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +52: [2023-04-29 09:45:08,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +26: [2023-04-29 09:45:08,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +22: [2023-04-29 09:45:08,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +57: [2023-04-29 09:45:08,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +22: [2023-04-29 09:45:08,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +59: [2023-04-29 09:45:08,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +22: [2023-04-29 09:45:08,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +22: [2023-04-29 09:45:08,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +42: [2023-04-29 09:45:08,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +57: [2023-04-29 09:45:08,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +57: [2023-04-29 09:45:08,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +35: [2023-04-29 09:45:08,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +50: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +50: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:08,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +40: [2023-04-29 09:45:08,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +46: [2023-04-29 09:45:08,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +46: [2023-04-29 09:45:08,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +35: [2023-04-29 09:45:08,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:08,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +40: [2023-04-29 09:45:08,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +40: [2023-04-29 09:45:08,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +57: [2023-04-29 09:45:08,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +44: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +62: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +61: [2023-04-29 09:45:08,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +35: [2023-04-29 09:45:08,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +61: [2023-04-29 09:45:08,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +61: [2023-04-29 09:45:08,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +62: [2023-04-29 09:45:08,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +50: [2023-04-29 09:45:08,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:08,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +56: [2023-04-29 09:45:08,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +57: [2023-04-29 09:45:08,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 6: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 6: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +35: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +62: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +47: [2023-04-29 09:45:08,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +26: [2023-04-29 09:45:08,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +57: [2023-04-29 09:45:08,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +26: [2023-04-29 09:45:08,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +61: [2023-04-29 09:45:08,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:08,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:08,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:08,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +56: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +57: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +50: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +20: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +20: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +20: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 4: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +57: [2023-04-29 09:45:08,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +20: [2023-04-29 09:45:08,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +57: [2023-04-29 09:45:08,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:08,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:08,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +57: [2023-04-29 09:45:08,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +50: [2023-04-29 09:45:08,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +61: [2023-04-29 09:45:08,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +56: [2023-04-29 09:45:08,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +40: [2023-04-29 09:45:08,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 4: [2023-04-29 09:45:08,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 4: [2023-04-29 09:45:08,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +56: [2023-04-29 09:45:08,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:08,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +53: [2023-04-29 09:45:08,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +29: [2023-04-29 09:45:08,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +21: [2023-04-29 09:45:08,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:08,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +53: [2023-04-29 09:45:08,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +40: [2023-04-29 09:45:08,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +59: [2023-04-29 09:45:08,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +59: [2023-04-29 09:45:08,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +47: [2023-04-29 09:45:08,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:08,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:08,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +61: [2023-04-29 09:45:08,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +42: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +42: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +42: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +62: [2023-04-29 09:45:08,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:08,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:08,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +47: [2023-04-29 09:45:08,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +42: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +49: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 1: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +59: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +60: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +42: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +29: [2023-04-29 09:45:08,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:08,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +21: [2023-04-29 09:45:08,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +21: [2023-04-29 09:45:08,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +60: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +29: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:08,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 7: [2023-04-29 09:45:08,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +20: [2023-04-29 09:45:08,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:08,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:08,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +42: [2023-04-29 09:45:08,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +62: [2023-04-29 09:45:08,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +50: [2023-04-29 09:45:08,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +62: [2023-04-29 09:45:08,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +29: [2023-04-29 09:45:08,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +22: [2023-04-29 09:45:08,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:08,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +21: [2023-04-29 09:45:08,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:08,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +52: [2023-04-29 09:45:08,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +20: [2023-04-29 09:45:08,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:08,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:08,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:08,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +41: [2023-04-29 09:45:08,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +20: [2023-04-29 09:45:08,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +22: [2023-04-29 09:45:08,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +41: [2023-04-29 09:45:08,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +62: [2023-04-29 09:45:08,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +41: [2023-04-29 09:45:08,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +61: [2023-04-29 09:45:08,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:08,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +22: [2023-04-29 09:45:08,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +22: [2023-04-29 09:45:08,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:08,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:08,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +55: [2023-04-29 09:45:08,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +55: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +55: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +55: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +61: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +55: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +55: [2023-04-29 09:45:08,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +49: [2023-04-29 09:45:08,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:08,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +47: [2023-04-29 09:45:08,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 6: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +44: [2023-04-29 09:45:08,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 2: [2023-04-29 09:45:08,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:08,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +33: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +33: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +25: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +49: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:08,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 1: [2023-04-29 09:45:08,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +47: [2023-04-29 09:45:08,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +44: [2023-04-29 09:45:08,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +47: [2023-04-29 09:45:08,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +44: [2023-04-29 09:45:08,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +60: [2023-04-29 09:45:08,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 6: [2023-04-29 09:45:08,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:08,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +25: [2023-04-29 09:45:08,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +50: [2023-04-29 09:45:08,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +21: [2023-04-29 09:45:08,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 7: [2023-04-29 09:45:08,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +59: [2023-04-29 09:45:08,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:08,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +48: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +48: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +25: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +25: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +44: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +48: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +44: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 1: [2023-04-29 09:45:08,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +21: [2023-04-29 09:45:08,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +40: [2023-04-29 09:45:08,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:08,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +29: [2023-04-29 09:45:08,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:08,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +25: [2023-04-29 09:45:08,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 7: [2023-04-29 09:45:08,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:08,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +46: [2023-04-29 09:45:08,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +25: [2023-04-29 09:45:08,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 2: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +49: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +25: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +52: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +40: [2023-04-29 09:45:08,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +25: [2023-04-29 09:45:08,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +22: [2023-04-29 09:45:08,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:08,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +24: [2023-04-29 09:45:08,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:08,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:08,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +50: [2023-04-29 09:45:08,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +29: [2023-04-29 09:45:08,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:08,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:08,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +62: [2023-04-29 09:45:08,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +44: [2023-04-29 09:45:08,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +24: [2023-04-29 09:45:08,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +44: [2023-04-29 09:45:08,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +52: [2023-04-29 09:45:08,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:08,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:08,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. + 3: [2023-04-29 09:45:08,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +10: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +51: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +51: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +46: [2023-04-29 09:45:08,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +10: [2023-04-29 09:45:08,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:08,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:08,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:08,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:08,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:08,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:08,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +41: [2023-04-29 09:45:08,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +44: [2023-04-29 09:45:08,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 3: [2023-04-29 09:45:08,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +44: [2023-04-29 09:45:08,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +40: [2023-04-29 09:45:08,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +62: [2023-04-29 09:45:08,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +22: [2023-04-29 09:45:08,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:08,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:08,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:08,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +10: [2023-04-29 09:45:08,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:08,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +10: [2023-04-29 09:45:08,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:08,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 3: [2023-04-29 09:45:08,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:08,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +61: [2023-04-29 09:45:08,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 5: [2023-04-29 09:45:08,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +22: [2023-04-29 09:45:08,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +47: [2023-04-29 09:45:08,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:08,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +40: [2023-04-29 09:45:08,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +47: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +47: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +48: [2023-04-29 09:45:08,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:08,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:08,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:08,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +45: [2023-04-29 09:45:08,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +45: [2023-04-29 09:45:08,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +45: [2023-04-29 09:45:08,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +45: [2023-04-29 09:45:08,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_30-model_01-model_states.pt. +62: [2023-04-29 09:45:08,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 5: [2023-04-29 09:45:08,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:08,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:08,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:08,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +33: [2023-04-29 09:45:08,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... + 9: [2023-04-29 09:45:08,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:08,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:08,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:08,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +40: [2023-04-29 09:45:08,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +61: [2023-04-29 09:45:08,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +61: [2023-04-29 09:45:08,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:08,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 0: [2023-04-29 09:45:08,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:08,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 0: [2023-04-29 09:45:08,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +48: [2023-04-29 09:45:09,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +48: [2023-04-29 09:45:09,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +48: [2023-04-29 09:45:09,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +54: [2023-04-29 09:45:09,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +54: [2023-04-29 09:45:09,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +54: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +37: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +37: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +37: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +12: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +12: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +37: [2023-04-29 09:45:09,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +37: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +37: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +12: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +40: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +46: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +40: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +12: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +54: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +61: [2023-04-29 09:45:09,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +61: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +58: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +37: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +34: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +37: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +34: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +30: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +48: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:09,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +51: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +12: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +40: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +46: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +16: [2023-04-29 09:45:09,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +37: [2023-04-29 09:45:09,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +37: [2023-04-29 09:45:09,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +37: [2023-04-29 09:45:09,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +16: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +20: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +16: [2023-04-29 09:45:09,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +58: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +30: [2023-04-29 09:45:09,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +48: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +48: [2023-04-29 09:45:09,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +30: [2023-04-29 09:45:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +51: [2023-04-29 09:45:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +16: [2023-04-29 09:45:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +30: [2023-04-29 09:45:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +48: [2023-04-29 09:45:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +51: [2023-04-29 09:45:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +43: [2023-04-29 09:45:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +16: [2023-04-29 09:45:09,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +45: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +45: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +16: [2023-04-29 09:45:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +40: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +11: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +40: [2023-04-29 09:45:09,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +16: [2023-04-29 09:45:09,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +45: [2023-04-29 09:45:09,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +45: [2023-04-29 09:45:09,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt... +62: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +55: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +55: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +20: [2023-04-29 09:45:09,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +11: [2023-04-29 09:45:09,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +63: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +63: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +63: [2023-04-29 09:45:09,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +11: [2023-04-29 09:45:09,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +11: [2023-04-29 09:45:09,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +63: [2023-04-29 09:45:09,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +11: [2023-04-29 09:45:09,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:09,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +16: [2023-04-29 09:45:09,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +11: [2023-04-29 09:45:09,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +11: [2023-04-29 09:45:09,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +11: [2023-04-29 09:45:09,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +20: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +60: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +11: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +40: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +44: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +16: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +16: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +20: [2023-04-29 09:45:09,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:09,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +25: [2023-04-29 09:45:09,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +54: [2023-04-29 09:45:09,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +15: [2023-04-29 09:45:09,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +15: [2023-04-29 09:45:09,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +44: [2023-04-29 09:45:09,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:09,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:09,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:09,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:09,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +20: [2023-04-29 09:45:09,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +44: [2023-04-29 09:45:09,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:09,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +37: [2023-04-29 09:45:09,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +12: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +12: [2023-04-29 09:45:09,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +15: [2023-04-29 09:45:09,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +20: [2023-04-29 09:45:09,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +15: [2023-04-29 09:45:09,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +24: [2023-04-29 09:45:09,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +20: [2023-04-29 09:45:09,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +37: [2023-04-29 09:45:09,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:09,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +60: [2023-04-29 09:45:09,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +12: [2023-04-29 09:45:09,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +12: [2023-04-29 09:45:09,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:09,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +32: [2023-04-29 09:45:09,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +32: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +25: [2023-04-29 09:45:09,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +60: [2023-04-29 09:45:09,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +24: [2023-04-29 09:45:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +24: [2023-04-29 09:45:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +24: [2023-04-29 09:45:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +47: [2023-04-29 09:45:09,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +60: [2023-04-29 09:45:09,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +25: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +41: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +23: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +10: [2023-04-29 09:45:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +32: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +32: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +32: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +32: [2023-04-29 09:45:09,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +63: [2023-04-29 09:45:09,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:09,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +63: [2023-04-29 09:45:09,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:09,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:09,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +44: [2023-04-29 09:45:09,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:09,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +41: [2023-04-29 09:45:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +23: [2023-04-29 09:45:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +25: [2023-04-29 09:45:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +55: [2023-04-29 09:45:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +23: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +19: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +19: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +47: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +19: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +15: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +19: [2023-04-29 09:45:09,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +23: [2023-04-29 09:45:09,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +44: [2023-04-29 09:45:09,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +23: [2023-04-29 09:45:09,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +23: [2023-04-29 09:45:09,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +46: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +63: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +25: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +55: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +41: [2023-04-29 09:45:09,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +47: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +24: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +55: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +55: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +15: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +16: [2023-04-29 09:45:09,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +15: [2023-04-29 09:45:09,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +15: [2023-04-29 09:45:09,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +47: [2023-04-29 09:45:09,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +10: [2023-04-29 09:45:09,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:09,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +46: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +33: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +58: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +33: [2023-04-29 09:45:09,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +41: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +10: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +31: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +39: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +39: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +47: [2023-04-29 09:45:09,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +31: [2023-04-29 09:45:09,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +12: [2023-04-29 09:45:09,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:09,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +31: [2023-04-29 09:45:09,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +47: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +39: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +16: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +17: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +39: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +33: [2023-04-29 09:45:09,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +51: [2023-04-29 09:45:09,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +12: [2023-04-29 09:45:09,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +58: [2023-04-29 09:45:09,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +17: [2023-04-29 09:45:09,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +48: [2023-04-29 09:45:09,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +17: [2023-04-29 09:45:09,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +45: [2023-04-29 09:45:09,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +16: [2023-04-29 09:45:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +31: [2023-04-29 09:45:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +31: [2023-04-29 09:45:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +31: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +31: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +39: [2023-04-29 09:45:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +39: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +46: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +46: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +33: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +39: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +48: [2023-04-29 09:45:09,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +48: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +47: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +58: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +17: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +17: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +33: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +25: [2023-04-29 09:45:09,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:09,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +17: [2023-04-29 09:45:09,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +48: [2023-04-29 09:45:09,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 4: [2023-04-29 09:45:09,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +48: [2023-04-29 09:45:09,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:09,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:09,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +33: [2023-04-29 09:45:09,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +48: [2023-04-29 09:45:09,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +58: [2023-04-29 09:45:09,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +46: [2023-04-29 09:45:09,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +10: [2023-04-29 09:45:09,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +30: [2023-04-29 09:45:09,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +51: [2023-04-29 09:45:09,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +25: [2023-04-29 09:45:09,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +34: [2023-04-29 09:45:09,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +51: [2023-04-29 09:45:09,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +51: [2023-04-29 09:45:09,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +19: [2023-04-29 09:45:09,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:09,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +12: [2023-04-29 09:45:09,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +46: [2023-04-29 09:45:09,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +48: [2023-04-29 09:45:09,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +45: [2023-04-29 09:45:09,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +48: [2023-04-29 09:45:09,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +51: [2023-04-29 09:45:09,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +12: [2023-04-29 09:45:09,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +51: [2023-04-29 09:45:09,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:09,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +12: [2023-04-29 09:45:09,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +43: [2023-04-29 09:45:09,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +51: [2023-04-29 09:45:09,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +37: [2023-04-29 09:45:09,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +20: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +20: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +20: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +20: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +19: [2023-04-29 09:45:09,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +37: [2023-04-29 09:45:09,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +37: [2023-04-29 09:45:09,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +37: [2023-04-29 09:45:09,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +19: [2023-04-29 09:45:09,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +45: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +19: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +16: [2023-04-29 09:45:09,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +20: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +37: [2023-04-29 09:45:09,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +48: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +11: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +10: [2023-04-29 09:45:09,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:09,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +20: [2023-04-29 09:45:09,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +37: [2023-04-29 09:45:09,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +20: [2023-04-29 09:45:09,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:09,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:09,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:09,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:09,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +16: [2023-04-29 09:45:09,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +51: [2023-04-29 09:45:09,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +19: [2023-04-29 09:45:09,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:09,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +30: [2023-04-29 09:45:09,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +19: [2023-04-29 09:45:09,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +15: [2023-04-29 09:45:09,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:09,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +45: [2023-04-29 09:45:09,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +15: [2023-04-29 09:45:09,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:09,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +48: [2023-04-29 09:45:09,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:09,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +34: [2023-04-29 09:45:09,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +35: [2023-04-29 09:45:09,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +18: [2023-04-29 09:45:09,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +34: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +26: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +26: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +26: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +26: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +23: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +28: [2023-04-29 09:45:09,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +15: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:09,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +15: [2023-04-29 09:45:09,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +30: [2023-04-29 09:45:09,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:09,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +37: [2023-04-29 09:45:09,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +37: [2023-04-29 09:45:09,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +12: [2023-04-29 09:45:09,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +35: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +15: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +16: [2023-04-29 09:45:09,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +35: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +35: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +12: [2023-04-29 09:45:09,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +32: [2023-04-29 09:45:09,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +36: [2023-04-29 09:45:09,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +36: [2023-04-29 09:45:09,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +36: [2023-04-29 09:45:09,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +36: [2023-04-29 09:45:09,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +12: [2023-04-29 09:45:09,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:09,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_00-model_states.pt. +12: [2023-04-29 09:45:09,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:09,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +35: [2023-04-29 09:45:09,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:09,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +58: [2023-04-29 09:45:09,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +35: [2023-04-29 09:45:09,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +37: [2023-04-29 09:45:09,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +16: [2023-04-29 09:45:09,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +37: [2023-04-29 09:45:09,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +31: [2023-04-29 09:45:09,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +58: [2023-04-29 09:45:09,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +37: [2023-04-29 09:45:09,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +53: [2023-04-29 09:45:09,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +11: [2023-04-29 09:45:09,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +11: [2023-04-29 09:45:09,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +23: [2023-04-29 09:45:09,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +16: [2023-04-29 09:45:09,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +37: [2023-04-29 09:45:09,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt... +32: [2023-04-29 09:45:09,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +34: [2023-04-29 09:45:09,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +34: [2023-04-29 09:45:09,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +34: [2023-04-29 09:45:09,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +15: [2023-04-29 09:45:09,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:09,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:09,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +17: [2023-04-29 09:45:09,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +53: [2023-04-29 09:45:09,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +31: [2023-04-29 09:45:09,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +37: [2023-04-29 09:45:09,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +15: [2023-04-29 09:45:09,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +15: [2023-04-29 09:45:09,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +15: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +39: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +31: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +31: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +32: [2023-04-29 09:45:09,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +15: [2023-04-29 09:45:09,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:09,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +45: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +12: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +43: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +43: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +43: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +43: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +53: [2023-04-29 09:45:09,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +17: [2023-04-29 09:45:09,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:09,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:09,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +12: [2023-04-29 09:45:09,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +39: [2023-04-29 09:45:09,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +26: [2023-04-29 09:45:09,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +15: [2023-04-29 09:45:09,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +26: [2023-04-29 09:45:09,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +12: [2023-04-29 09:45:09,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +12: [2023-04-29 09:45:09,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +16: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +28: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +39: [2023-04-29 09:45:09,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +12: [2023-04-29 09:45:09,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +39: [2023-04-29 09:45:09,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +36: [2023-04-29 09:45:09,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:09,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +38: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +38: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +38: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +58: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +12: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +38: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +28: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +39: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +32: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +36: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +28: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +28: [2023-04-29 09:45:09,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +51: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +10: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +10: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +10: [2023-04-29 09:45:09,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +28: [2023-04-29 09:45:09,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +19: [2023-04-29 09:45:09,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +58: [2023-04-29 09:45:09,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +16: [2023-04-29 09:45:09,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:09,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +32: [2023-04-29 09:45:09,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +12: [2023-04-29 09:45:09,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +23: [2023-04-29 09:45:09,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +51: [2023-04-29 09:45:09,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +23: [2023-04-29 09:45:09,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +16: [2023-04-29 09:45:09,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +23: [2023-04-29 09:45:09,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +28: [2023-04-29 09:45:09,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +34: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:09,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:09,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +17: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +41: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +41: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +16: [2023-04-29 09:45:09,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +31: [2023-04-29 09:45:09,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +17: [2023-04-29 09:45:09,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +16: [2023-04-29 09:45:09,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +32: [2023-04-29 09:45:09,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +31: [2023-04-29 09:45:09,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +28: [2023-04-29 09:45:09,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +28: [2023-04-29 09:45:09,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:09,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +16: [2023-04-29 09:45:09,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +43: [2023-04-29 09:45:09,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +18: [2023-04-29 09:45:09,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:09,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +22: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +22: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +49: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +49: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +49: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +22: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +49: [2023-04-29 09:45:09,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +22: [2023-04-29 09:45:09,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +16: [2023-04-29 09:45:09,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +39: [2023-04-29 09:45:09,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +39: [2023-04-29 09:45:09,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +35: [2023-04-29 09:45:09,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +38: [2023-04-29 09:45:09,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +18: [2023-04-29 09:45:09,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +10: [2023-04-29 09:45:09,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +38: [2023-04-29 09:45:09,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +38: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +38: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +27: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +27: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +20: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +10: [2023-04-29 09:45:09,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:09,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +51: [2023-04-29 09:45:09,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +15: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +15: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +60: [2023-04-29 09:45:09,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +15: [2023-04-29 09:45:09,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +20: [2023-04-29 09:45:09,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +20: [2023-04-29 09:45:09,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +51: [2023-04-29 09:45:09,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:09,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +56: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +56: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +10: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +56: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +60: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +15: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +56: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +60: [2023-04-29 09:45:09,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +32: [2023-04-29 09:45:09,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +10: [2023-04-29 09:45:09,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:09,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +35: [2023-04-29 09:45:09,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +19: [2023-04-29 09:45:09,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +51: [2023-04-29 09:45:09,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +19: [2023-04-29 09:45:09,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:09,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +35: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +63: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +35: [2023-04-29 09:45:09,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 9: [2023-04-29 09:45:09,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +18: [2023-04-29 09:45:09,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 9: [2023-04-29 09:45:09,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +63: [2023-04-29 09:45:09,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +63: [2023-04-29 09:45:09,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:09,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:09,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +19: [2023-04-29 09:45:09,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:09,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +63: [2023-04-29 09:45:09,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +63: [2023-04-29 09:45:09,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +63: [2023-04-29 09:45:09,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +22: [2023-04-29 09:45:09,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +49: [2023-04-29 09:45:09,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +63: [2023-04-29 09:45:09,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +22: [2023-04-29 09:45:09,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +49: [2023-04-29 09:45:09,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +63: [2023-04-29 09:45:09,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +19: [2023-04-29 09:45:09,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +19: [2023-04-29 09:45:09,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +63: [2023-04-29 09:45:09,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +22: [2023-04-29 09:45:09,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +26: [2023-04-29 09:45:09,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:09,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +49: [2023-04-29 09:45:09,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +22: [2023-04-29 09:45:09,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:09,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +35: [2023-04-29 09:45:09,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +13: [2023-04-29 09:45:09,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:09,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:09,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +54: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +27: [2023-04-29 09:45:09,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +57: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +26: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +38: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +38: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +13: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:09,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +38: [2023-04-29 09:45:09,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +56: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +27: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +56: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +15: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:09,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +18: [2023-04-29 09:45:09,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +18: [2023-04-29 09:45:09,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +18: [2023-04-29 09:45:09,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +57: [2023-04-29 09:45:09,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +18: [2023-04-29 09:45:09,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +18: [2023-04-29 09:45:09,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +57: [2023-04-29 09:45:09,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +57: [2023-04-29 09:45:09,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:09,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +15: [2023-04-29 09:45:09,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +57: [2023-04-29 09:45:09,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +27: [2023-04-29 09:45:09,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:09,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +38: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +38: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +18: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +50: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +54: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +38: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +28: [2023-04-29 09:45:09,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +26: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +54: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +54: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +54: [2023-04-29 09:45:09,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +20: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +27: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +56: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +56: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +42: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +26: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +36: [2023-04-29 09:45:09,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +58: [2023-04-29 09:45:09,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +28: [2023-04-29 09:45:09,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +58: [2023-04-29 09:45:09,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +19: [2023-04-29 09:45:09,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +35: [2023-04-29 09:45:09,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +26: [2023-04-29 09:45:09,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +58: [2023-04-29 09:45:09,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +41: [2023-04-29 09:45:09,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +42: [2023-04-29 09:45:09,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +58: [2023-04-29 09:45:09,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +42: [2023-04-29 09:45:09,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +36: [2023-04-29 09:45:09,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +27: [2023-04-29 09:45:09,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +58: [2023-04-29 09:45:09,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +42: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +56: [2023-04-29 09:45:09,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +56: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +36: [2023-04-29 09:45:09,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +42: [2023-04-29 09:45:09,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +34: [2023-04-29 09:45:09,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +34: [2023-04-29 09:45:09,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +21: [2023-04-29 09:45:09,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +21: [2023-04-29 09:45:09,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +43: [2023-04-29 09:45:09,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +21: [2023-04-29 09:45:09,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +21: [2023-04-29 09:45:09,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +34: [2023-04-29 09:45:09,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +34: [2023-04-29 09:45:09,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +39: [2023-04-29 09:45:09,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +19: [2023-04-29 09:45:09,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +34: [2023-04-29 09:45:09,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +19: [2023-04-29 09:45:09,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +36: [2023-04-29 09:45:09,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +34: [2023-04-29 09:45:09,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +36: [2023-04-29 09:45:09,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +39: [2023-04-29 09:45:09,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +18: [2023-04-29 09:45:09,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +36: [2023-04-29 09:45:09,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +18: [2023-04-29 09:45:09,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:09,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:09,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +39: [2023-04-29 09:45:09,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +18: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +43: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +18: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +10: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +25: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +39: [2023-04-29 09:45:09,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +38: [2023-04-29 09:45:09,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +38: [2023-04-29 09:45:09,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +25: [2023-04-29 09:45:09,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:09,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +43: [2023-04-29 09:45:09,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +27: [2023-04-29 09:45:09,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +28: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +49: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +10: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +39: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +49: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +39: [2023-04-29 09:45:09,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +10: [2023-04-29 09:45:09,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +39: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +28: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +38: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +38: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +38: [2023-04-29 09:45:09,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +43: [2023-04-29 09:45:09,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +28: [2023-04-29 09:45:09,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +49: [2023-04-29 09:45:09,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +43: [2023-04-29 09:45:09,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +43: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +41: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +49: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +10: [2023-04-29 09:45:09,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +60: [2023-04-29 09:45:09,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +24: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +49: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +29: [2023-04-29 09:45:09,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +24: [2023-04-29 09:45:09,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +60: [2023-04-29 09:45:09,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +29: [2023-04-29 09:45:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +29: [2023-04-29 09:45:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +29: [2023-04-29 09:45:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +22: [2023-04-29 09:45:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +27: [2023-04-29 09:45:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +49: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +38: [2023-04-29 09:45:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +38: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +28: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +28: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +10: [2023-04-29 09:45:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +21: [2023-04-29 09:45:09,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +35: [2023-04-29 09:45:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +35: [2023-04-29 09:45:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +10: [2023-04-29 09:45:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +61: [2023-04-29 09:45:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +22: [2023-04-29 09:45:09,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +28: [2023-04-29 09:45:09,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +60: [2023-04-29 09:45:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +21: [2023-04-29 09:45:09,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +28: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +14: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +14: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +14: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +14: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +63: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +21: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +35: [2023-04-29 09:45:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +35: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +63: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +10: [2023-04-29 09:45:09,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +13: [2023-04-29 09:45:09,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:09,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +61: [2023-04-29 09:45:09,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +10: [2023-04-29 09:45:09,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +36: [2023-04-29 09:45:09,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +61: [2023-04-29 09:45:09,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +22: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +36: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +61: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +63: [2023-04-29 09:45:09,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +63: [2023-04-29 09:45:09,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +13: [2023-04-29 09:45:09,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +13: [2023-04-29 09:45:09,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +62: [2023-04-29 09:45:09,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +62: [2023-04-29 09:45:09,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +62: [2023-04-29 09:45:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +22: [2023-04-29 09:45:09,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +62: [2023-04-29 09:45:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +17: [2023-04-29 09:45:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +17: [2023-04-29 09:45:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +13: [2023-04-29 09:45:09,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:09,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +13: [2023-04-29 09:45:09,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +25: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +49: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +49: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +49: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +27: [2023-04-29 09:45:09,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +57: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +57: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +56: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +22: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +38: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +17: [2023-04-29 09:45:09,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +60: [2023-04-29 09:45:09,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:09,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +17: [2023-04-29 09:45:09,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +50: [2023-04-29 09:45:09,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +24: [2023-04-29 09:45:09,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:09,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +38: [2023-04-29 09:45:09,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +38: [2023-04-29 09:45:09,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +27: [2023-04-29 09:45:09,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +60: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +25: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +22: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +56: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +27: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +56: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +56: [2023-04-29 09:45:09,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +29: [2023-04-29 09:45:09,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +52: [2023-04-29 09:45:09,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +52: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +52: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +56: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +57: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +29: [2023-04-29 09:45:09,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:09,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +29: [2023-04-29 09:45:09,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +29: [2023-04-29 09:45:09,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +56: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +35: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +35: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +24: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +42: [2023-04-29 09:45:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +27: [2023-04-29 09:45:09,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +14: [2023-04-29 09:45:09,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +28: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +59: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +63: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +13: [2023-04-29 09:45:09,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +36: [2023-04-29 09:45:09,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:09,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +60: [2023-04-29 09:45:09,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:09,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +54: [2023-04-29 09:45:09,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +56: [2023-04-29 09:45:09,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +56: [2023-04-29 09:45:09,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +63: [2023-04-29 09:45:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +35: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 9: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +35: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +59: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +27: [2023-04-29 09:45:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +36: [2023-04-29 09:45:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +59: [2023-04-29 09:45:09,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +42: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +36: [2023-04-29 09:45:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +53: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +53: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +53: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +53: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +36: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 6: [2023-04-29 09:45:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +63: [2023-04-29 09:45:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +27: [2023-04-29 09:45:09,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +17: [2023-04-29 09:45:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +17: [2023-04-29 09:45:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +57: [2023-04-29 09:45:09,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +62: [2023-04-29 09:45:09,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +62: [2023-04-29 09:45:09,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +56: [2023-04-29 09:45:09,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +14: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +56: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +13: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +62: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +13: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 4: [2023-04-29 09:45:09,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +62: [2023-04-29 09:45:09,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +18: [2023-04-29 09:45:09,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +38: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +18: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +56: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +57: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +27: [2023-04-29 09:45:09,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +38: [2023-04-29 09:45:09,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +17: [2023-04-29 09:45:09,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:09,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:09,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +50: [2023-04-29 09:45:09,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +14: [2023-04-29 09:45:09,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +56: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +13: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +38: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +13: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +52: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +56: [2023-04-29 09:45:09,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +17: [2023-04-29 09:45:09,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +57: [2023-04-29 09:45:09,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +57: [2023-04-29 09:45:09,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +18: [2023-04-29 09:45:09,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +18: [2023-04-29 09:45:09,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +57: [2023-04-29 09:45:09,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +18: [2023-04-29 09:45:09,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +18: [2023-04-29 09:45:09,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:09,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:09,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +42: [2023-04-29 09:45:09,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:09,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +14: [2023-04-29 09:45:09,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +49: [2023-04-29 09:45:09,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +57: [2023-04-29 09:45:09,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +52: [2023-04-29 09:45:09,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +14: [2023-04-29 09:45:09,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +57: [2023-04-29 09:45:09,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:09,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +33: [2023-04-29 09:45:09,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +52: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +47: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +47: [2023-04-29 09:45:09,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +18: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +47: [2023-04-29 09:45:09,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +18: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +47: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:09,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:09,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:09,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:09,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:09,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +42: [2023-04-29 09:45:09,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +49: [2023-04-29 09:45:09,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +49: [2023-04-29 09:45:09,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:09,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:09,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +47: [2023-04-29 09:45:09,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:09,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +54: [2023-04-29 09:45:09,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:09,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +59: [2023-04-29 09:45:09,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:09,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +20: [2023-04-29 09:45:09,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +20: [2023-04-29 09:45:09,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +20: [2023-04-29 09:45:09,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +20: [2023-04-29 09:45:09,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +56: [2023-04-29 09:45:09,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:09,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +42: [2023-04-29 09:45:09,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +42: [2023-04-29 09:45:09,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +42: [2023-04-29 09:45:09,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +30: [2023-04-29 09:45:09,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +30: [2023-04-29 09:45:09,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +56: [2023-04-29 09:45:09,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +49: [2023-04-29 09:45:09,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +11: [2023-04-29 09:45:09,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +11: [2023-04-29 09:45:09,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +11: [2023-04-29 09:45:09,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +11: [2023-04-29 09:45:09,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +30: [2023-04-29 09:45:09,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +30: [2023-04-29 09:45:09,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +24: [2023-04-29 09:45:09,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:09,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +40: [2023-04-29 09:45:09,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +40: [2023-04-29 09:45:09,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +40: [2023-04-29 09:45:09,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +50: [2023-04-29 09:45:09,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:09,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +61: [2023-04-29 09:45:09,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +55: [2023-04-29 09:45:09,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +61: [2023-04-29 09:45:09,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +40: [2023-04-29 09:45:09,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +29: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +55: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:09,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +44: [2023-04-29 09:45:09,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +44: [2023-04-29 09:45:09,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +40: [2023-04-29 09:45:09,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +40: [2023-04-29 09:45:09,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +50: [2023-04-29 09:45:09,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +50: [2023-04-29 09:45:09,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:09,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:09,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +55: [2023-04-29 09:45:09,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +55: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +44: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +44: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +35: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +35: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +60: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +55: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +55: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:09,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +61: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +55: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +29: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +55: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +50: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +24: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +62: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +44: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +14: [2023-04-29 09:45:09,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:09,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +44: [2023-04-29 09:45:09,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +13: [2023-04-29 09:45:09,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:09,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +20: [2023-04-29 09:45:09,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +49: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +44: [2023-04-29 09:45:09,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +35: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +35: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +49: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +14: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +13: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +44: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +24: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +44: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +42: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:09,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +29: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +29: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +17: [2023-04-29 09:45:09,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +59: [2023-04-29 09:45:09,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +49: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +20: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +59: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +20: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +35: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +24: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +42: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:09,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +11: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +42: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +30: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +62: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +30: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +44: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +17: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +11: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +44: [2023-04-29 09:45:09,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +13: [2023-04-29 09:45:09,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +20: [2023-04-29 09:45:09,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +11: [2023-04-29 09:45:09,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +17: [2023-04-29 09:45:09,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +11: [2023-04-29 09:45:09,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +13: [2023-04-29 09:45:09,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +35: [2023-04-29 09:45:09,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +17: [2023-04-29 09:45:09,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +35: [2023-04-29 09:45:09,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +14: [2023-04-29 09:45:09,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +40: [2023-04-29 09:45:09,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +40: [2023-04-29 09:45:09,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +13: [2023-04-29 09:45:09,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +62: [2023-04-29 09:45:09,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:09,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +61: [2023-04-29 09:45:09,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +17: [2023-04-29 09:45:09,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +35: [2023-04-29 09:45:09,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +13: [2023-04-29 09:45:09,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +62: [2023-04-29 09:45:09,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +57: [2023-04-29 09:45:09,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +17: [2023-04-29 09:45:09,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:09,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:09,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +29: [2023-04-29 09:45:09,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +14: [2023-04-29 09:45:09,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:09,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +13: [2023-04-29 09:45:09,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +17: [2023-04-29 09:45:09,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +60: [2023-04-29 09:45:09,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +60: [2023-04-29 09:45:09,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +40: [2023-04-29 09:45:09,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +55: [2023-04-29 09:45:09,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:09,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +44: [2023-04-29 09:45:09,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:09,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +61: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +61: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +55: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +57: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +57: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +57: [2023-04-29 09:45:09,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +33: [2023-04-29 09:45:09,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +29: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +52: [2023-04-29 09:45:09,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +17: [2023-04-29 09:45:09,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:09,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +29: [2023-04-29 09:45:09,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +29: [2023-04-29 09:45:09,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +47: [2023-04-29 09:45:09,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +47: [2023-04-29 09:45:09,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +14: [2023-04-29 09:45:09,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +14: [2023-04-29 09:45:09,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +60: [2023-04-29 09:45:09,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +57: [2023-04-29 09:45:09,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +62: [2023-04-29 09:45:09,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +14: [2023-04-29 09:45:09,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +61: [2023-04-29 09:45:09,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +59: [2023-04-29 09:45:09,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +29: [2023-04-29 09:45:09,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +14: [2023-04-29 09:45:09,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:09,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +47: [2023-04-29 09:45:09,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:09,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:09,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +62: [2023-04-29 09:45:09,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:09,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +53: [2023-04-29 09:45:09,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +52: [2023-04-29 09:45:09,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +52: [2023-04-29 09:45:09,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +59: [2023-04-29 09:45:09,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +47: [2023-04-29 09:45:09,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +53: [2023-04-29 09:45:09,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +53: [2023-04-29 09:45:09,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +14: [2023-04-29 09:45:09,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:09,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +52: [2023-04-29 09:45:09,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +52: [2023-04-29 09:45:09,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +14: [2023-04-29 09:45:09,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +59: [2023-04-29 09:45:09,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +53: [2023-04-29 09:45:09,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +50: [2023-04-29 09:45:09,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +59: [2023-04-29 09:45:09,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +53: [2023-04-29 09:45:09,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +33: [2023-04-29 09:45:09,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +59: [2023-04-29 09:45:09,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +50: [2023-04-29 09:45:09,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +47: [2023-04-29 09:45:09,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:09,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +47: [2023-04-29 09:45:09,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +47: [2023-04-29 09:45:09,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:09,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +23: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +62: [2023-04-29 09:45:09,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +23: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +23: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +23: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +15: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +46: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +15: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +15: [2023-04-29 09:45:09,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +15: [2023-04-29 09:45:09,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +50: [2023-04-29 09:45:09,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +47: [2023-04-29 09:45:09,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:09,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +55: [2023-04-29 09:45:09,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +30: [2023-04-29 09:45:09,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +50: [2023-04-29 09:45:09,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +42: [2023-04-29 09:45:09,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +42: [2023-04-29 09:45:09,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +50: [2023-04-29 09:45:09,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +42: [2023-04-29 09:45:09,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +42: [2023-04-29 09:45:09,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +20: [2023-04-29 09:45:09,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +20: [2023-04-29 09:45:09,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:09,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +30: [2023-04-29 09:45:09,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +59: [2023-04-29 09:45:09,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +14: [2023-04-29 09:45:09,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:09,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +42: [2023-04-29 09:45:09,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +42: [2023-04-29 09:45:09,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +40: [2023-04-29 09:45:09,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +50: [2023-04-29 09:45:09,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +44: [2023-04-29 09:45:09,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +44: [2023-04-29 09:45:09,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +14: [2023-04-29 09:45:09,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:09,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +26: [2023-04-29 09:45:09,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +44: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +41: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +41: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +20: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +61: [2023-04-29 09:45:09,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +41: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +20: [2023-04-29 09:45:09,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +16: [2023-04-29 09:45:09,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +16: [2023-04-29 09:45:09,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +16: [2023-04-29 09:45:09,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +16: [2023-04-29 09:45:09,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +30: [2023-04-29 09:45:09,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +30: [2023-04-29 09:45:09,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +40: [2023-04-29 09:45:09,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +59: [2023-04-29 09:45:09,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +61: [2023-04-29 09:45:09,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +26: [2023-04-29 09:45:09,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +31: [2023-04-29 09:45:09,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +62: [2023-04-29 09:45:09,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +44: [2023-04-29 09:45:09,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +62: [2023-04-29 09:45:09,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +20: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +22: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +29: [2023-04-29 09:45:09,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +61: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +22: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +22: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_09-model_01-model_states.pt. +26: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +61: [2023-04-29 09:45:09,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:09,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +26: [2023-04-29 09:45:09,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +26: [2023-04-29 09:45:09,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +40: [2023-04-29 09:45:09,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +40: [2023-04-29 09:45:09,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +60: [2023-04-29 09:45:09,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +22: [2023-04-29 09:45:09,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +29: [2023-04-29 09:45:09,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +61: [2023-04-29 09:45:09,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +22: [2023-04-29 09:45:09,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 6: [2023-04-29 09:45:09,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +62: [2023-04-29 09:45:09,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +22: [2023-04-29 09:45:09,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +12: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +12: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +55: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +44: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +44: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 6: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +15: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +12: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +15: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +44: [2023-04-29 09:45:09,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +15: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +60: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +40: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +60: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +45: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +45: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +45: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +51: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +21: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +45: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +40: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +52: [2023-04-29 09:45:09,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +55: [2023-04-29 09:45:09,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +55: [2023-04-29 09:45:09,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +47: [2023-04-29 09:45:09,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +47: [2023-04-29 09:45:09,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +47: [2023-04-29 09:45:09,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +47: [2023-04-29 09:45:09,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +59: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +51: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +46: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +46: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +61: [2023-04-29 09:45:09,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +33: [2023-04-29 09:45:09,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +44: [2023-04-29 09:45:09,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +52: [2023-04-29 09:45:09,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +60: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +52: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +21: [2023-04-29 09:45:09,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +21: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +33: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +29: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +21: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +21: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +44: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +44: [2023-04-29 09:45:09,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +21: [2023-04-29 09:45:09,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +21: [2023-04-29 09:45:09,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +59: [2023-04-29 09:45:09,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +44: [2023-04-29 09:45:09,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +55: [2023-04-29 09:45:09,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +21: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +55: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +55: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +55: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +55: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +55: [2023-04-29 09:45:09,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +52: [2023-04-29 09:45:09,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +55: [2023-04-29 09:45:09,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +46: [2023-04-29 09:45:09,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_31-model_01-model_states.pt. +29: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +44: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +44: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +40: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:09,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +40: [2023-04-29 09:45:09,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +59: [2023-04-29 09:45:09,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +44: [2023-04-29 09:45:09,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +16: [2023-04-29 09:45:09,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:09,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +31: [2023-04-29 09:45:09,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +31: [2023-04-29 09:45:09,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +40: [2023-04-29 09:45:09,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +16: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +29: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +44: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +41: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +40: [2023-04-29 09:45:09,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +31: [2023-04-29 09:45:09,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt... +29: [2023-04-29 09:45:09,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:09,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:09,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:09,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +40: [2023-04-29 09:45:09,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:09,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... + 9: [2023-04-29 09:45:09,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:09,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +45: [2023-04-29 09:45:09,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +47: [2023-04-29 09:45:09,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +51: [2023-04-29 09:45:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +47: [2023-04-29 09:45:09,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +47: [2023-04-29 09:45:09,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +47: [2023-04-29 09:45:09,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +51: [2023-04-29 09:45:09,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:09,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +51: [2023-04-29 09:45:09,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +46: [2023-04-29 09:45:09,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:09,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +30: [2023-04-29 09:45:09,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +52: [2023-04-29 09:45:09,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +52: [2023-04-29 09:45:09,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:09,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +33: [2023-04-29 09:45:09,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:09,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +33: [2023-04-29 09:45:09,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt... +29: [2023-04-29 09:45:09,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:09,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +26: [2023-04-29 09:45:09,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +22: [2023-04-29 09:45:09,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +25: [2023-04-29 09:45:09,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:09,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +21: [2023-04-29 09:45:09,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +25: [2023-04-29 09:45:09,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +25: [2023-04-29 09:45:09,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +21: [2023-04-29 09:45:09,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +25: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +19: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +19: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +41: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +25: [2023-04-29 09:45:09,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +19: [2023-04-29 09:45:09,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:09,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +41: [2023-04-29 09:45:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +16: [2023-04-29 09:45:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +32: [2023-04-29 09:45:09,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +32: [2023-04-29 09:45:09,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +32: [2023-04-29 09:45:09,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +32: [2023-04-29 09:45:09,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +31: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +48: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +48: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +31: [2023-04-29 09:45:09,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +45: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +12: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +16: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +26: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +33: [2023-04-29 09:45:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +16: [2023-04-29 09:45:09,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_00-model_states.pt. +26: [2023-04-29 09:45:09,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:09,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +31: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +41: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +33: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +12: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +23: [2023-04-29 09:45:09,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +22: [2023-04-29 09:45:09,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +47: [2023-04-29 09:45:09,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:09,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +22: [2023-04-29 09:45:09,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +41: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt... +45: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +12: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +45: [2023-04-29 09:45:09,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +12: [2023-04-29 09:45:09,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +17: [2023-04-29 09:45:09,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +17: [2023-04-29 09:45:09,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 6: [2023-04-29 09:45:09,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 6: [2023-04-29 09:45:09,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:09,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +17: [2023-04-29 09:45:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +17: [2023-04-29 09:45:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +47: [2023-04-29 09:45:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +47: [2023-04-29 09:45:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +21: [2023-04-29 09:45:09,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +45: [2023-04-29 09:45:09,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +45: [2023-04-29 09:45:09,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +12: [2023-04-29 09:45:09,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +12: [2023-04-29 09:45:09,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +51: [2023-04-29 09:45:09,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +51: [2023-04-29 09:45:09,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +31: [2023-04-29 09:45:09,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:09,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +17: [2023-04-29 09:45:09,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +17: [2023-04-29 09:45:09,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:09,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +46: [2023-04-29 09:45:09,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +51: [2023-04-29 09:45:09,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +33: [2023-04-29 09:45:09,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +19: [2023-04-29 09:45:09,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +47: [2023-04-29 09:45:09,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +46: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +15: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +47: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +47: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +46: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +15: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +33: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +32: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +31: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +10: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +10: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +15: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +15: [2023-04-29 09:45:09,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:09,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +51: [2023-04-29 09:45:09,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +51: [2023-04-29 09:45:09,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +32: [2023-04-29 09:45:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +32: [2023-04-29 09:45:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +19: [2023-04-29 09:45:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +12: [2023-04-29 09:45:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +15: [2023-04-29 09:45:09,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +51: [2023-04-29 09:45:09,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. +46: [2023-04-29 09:45:09,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +15: [2023-04-29 09:45:09,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +15: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +33: [2023-04-29 09:45:09,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +31: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +30: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +33: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +18: [2023-04-29 09:45:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +21: [2023-04-29 09:45:09,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:09,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +48: [2023-04-29 09:45:09,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +48: [2023-04-29 09:45:09,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +18: [2023-04-29 09:45:09,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +51: [2023-04-29 09:45:09,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +18: [2023-04-29 09:45:09,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +33: [2023-04-29 09:45:09,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt... +18: [2023-04-29 09:45:09,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 9: [2023-04-29 09:45:09,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 9: [2023-04-29 09:45:09,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 9: [2023-04-29 09:45:09,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +24: [2023-04-29 09:45:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +18: [2023-04-29 09:45:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +18: [2023-04-29 09:45:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +24: [2023-04-29 09:45:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +24: [2023-04-29 09:45:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +11: [2023-04-29 09:45:09,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:09,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +18: [2023-04-29 09:45:09,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:09,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:09,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +10: [2023-04-29 09:45:09,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +10: [2023-04-29 09:45:09,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +30: [2023-04-29 09:45:09,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +11: [2023-04-29 09:45:09,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +11: [2023-04-29 09:45:09,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:09,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:09,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +45: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +45: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +45: [2023-04-29 09:45:09,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:09,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +22: [2023-04-29 09:45:09,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +22: [2023-04-29 09:45:09,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +22: [2023-04-29 09:45:09,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +22: [2023-04-29 09:45:09,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +45: [2023-04-29 09:45:09,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +25: [2023-04-29 09:45:09,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +45: [2023-04-29 09:45:09,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +25: [2023-04-29 09:45:09,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +45: [2023-04-29 09:45:09,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:09,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +24: [2023-04-29 09:45:09,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:09,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +23: [2023-04-29 09:45:09,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +12: [2023-04-29 09:45:09,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +37: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +37: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +48: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +12: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +23: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +48: [2023-04-29 09:45:09,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +12: [2023-04-29 09:45:09,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +24: [2023-04-29 09:45:09,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +24: [2023-04-29 09:45:09,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +26: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +26: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +17: [2023-04-29 09:45:09,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +18: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +17: [2023-04-29 09:45:09,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +17: [2023-04-29 09:45:09,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +17: [2023-04-29 09:45:09,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +15: [2023-04-29 09:45:09,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +63: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +63: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +54: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +54: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +54: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +27: [2023-04-29 09:45:09,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +43: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +63: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +27: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +43: [2023-04-29 09:45:09,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +43: [2023-04-29 09:45:09,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +15: [2023-04-29 09:45:09,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +36: [2023-04-29 09:45:09,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +36: [2023-04-29 09:45:09,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +54: [2023-04-29 09:45:09,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +31: [2023-04-29 09:45:09,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +22: [2023-04-29 09:45:09,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +36: [2023-04-29 09:45:09,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +36: [2023-04-29 09:45:09,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +22: [2023-04-29 09:45:09,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +22: [2023-04-29 09:45:09,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +37: [2023-04-29 09:45:09,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +23: [2023-04-29 09:45:09,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +23: [2023-04-29 09:45:09,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:09,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +13: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +12: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +23: [2023-04-29 09:45:09,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:09,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:09,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:09,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +48: [2023-04-29 09:45:09,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +53: [2023-04-29 09:45:09,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +53: [2023-04-29 09:45:09,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +53: [2023-04-29 09:45:09,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:09,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +13: [2023-04-29 09:45:09,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +48: [2023-04-29 09:45:09,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +48: [2023-04-29 09:45:09,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +13: [2023-04-29 09:45:09,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:09,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +26: [2023-04-29 09:45:09,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +53: [2023-04-29 09:45:09,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +53: [2023-04-29 09:45:09,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +53: [2023-04-29 09:45:09,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +53: [2023-04-29 09:45:09,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:09,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +26: [2023-04-29 09:45:09,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:09,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +25: [2023-04-29 09:45:09,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:09,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +48: [2023-04-29 09:45:09,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +48: [2023-04-29 09:45:09,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +18: [2023-04-29 09:45:09,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +15: [2023-04-29 09:45:09,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +63: [2023-04-29 09:45:09,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +17: [2023-04-29 09:45:09,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:09,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:09,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +28: [2023-04-29 09:45:09,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +28: [2023-04-29 09:45:09,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +28: [2023-04-29 09:45:09,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +48: [2023-04-29 09:45:09,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +63: [2023-04-29 09:45:09,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:09,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +17: [2023-04-29 09:45:09,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +36: [2023-04-29 09:45:09,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +18: [2023-04-29 09:45:09,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +11: [2023-04-29 09:45:09,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +54: [2023-04-29 09:45:09,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +31: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +31: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +54: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:09,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:09,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +27: [2023-04-29 09:45:09,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +18: [2023-04-29 09:45:09,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:09,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +43: [2023-04-29 09:45:09,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:09,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +36: [2023-04-29 09:45:09,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:09,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:09,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +45: [2023-04-29 09:45:09,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +11: [2023-04-29 09:45:09,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +36: [2023-04-29 09:45:09,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +43: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +43: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +15: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +36: [2023-04-29 09:45:09,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:09,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:09,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +43: [2023-04-29 09:45:09,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +16: [2023-04-29 09:45:09,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +16: [2023-04-29 09:45:09,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +16: [2023-04-29 09:45:09,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +43: [2023-04-29 09:45:09,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +24: [2023-04-29 09:45:09,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +11: [2023-04-29 09:45:09,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:09,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:09,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:09,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:09,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:09,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +24: [2023-04-29 09:45:09,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +16: [2023-04-29 09:45:09,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:09,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +48: [2023-04-29 09:45:09,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:09,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:09,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:09,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +18: [2023-04-29 09:45:09,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +27: [2023-04-29 09:45:09,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:09,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 9: [2023-04-29 09:45:09,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +58: [2023-04-29 09:45:09,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:09,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +48: [2023-04-29 09:45:09,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +45: [2023-04-29 09:45:09,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 9: [2023-04-29 09:45:09,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +32: [2023-04-29 09:45:09,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +18: [2023-04-29 09:45:09,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +32: [2023-04-29 09:45:09,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +32: [2023-04-29 09:45:09,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +32: [2023-04-29 09:45:09,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +32: [2023-04-29 09:45:09,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 9: [2023-04-29 09:45:09,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +32: [2023-04-29 09:45:09,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +32: [2023-04-29 09:45:09,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +32: [2023-04-29 09:45:09,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:09,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:09,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +11: [2023-04-29 09:45:09,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:09,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +29: [2023-04-29 09:45:09,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +29: [2023-04-29 09:45:09,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +58: [2023-04-29 09:45:09,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +29: [2023-04-29 09:45:09,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +45: [2023-04-29 09:45:09,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +18: [2023-04-29 09:45:09,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:09,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:09,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:09,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +48: [2023-04-29 09:45:09,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:09,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +13: [2023-04-29 09:45:09,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +22: [2023-04-29 09:45:09,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +13: [2023-04-29 09:45:09,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +22: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:09,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +22: [2023-04-29 09:45:09,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +22: [2023-04-29 09:45:09,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +22: [2023-04-29 09:45:09,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +18: [2023-04-29 09:45:09,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +48: [2023-04-29 09:45:09,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:09,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +10: [2023-04-29 09:45:09,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:09,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +28: [2023-04-29 09:45:09,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +28: [2023-04-29 09:45:09,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +14: [2023-04-29 09:45:09,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +18: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 2: [2023-04-29 09:45:09,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +14: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +19: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +18: [2023-04-29 09:45:09,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:09,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +28: [2023-04-29 09:45:09,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:09,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:09,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +10: [2023-04-29 09:45:09,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +26: [2023-04-29 09:45:09,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +10: [2023-04-29 09:45:09,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +53: [2023-04-29 09:45:09,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +53: [2023-04-29 09:45:09,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +10: [2023-04-29 09:45:09,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:09,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +19: [2023-04-29 09:45:09,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +28: [2023-04-29 09:45:09,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +28: [2023-04-29 09:45:09,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:09,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +53: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +10: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +18: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +53: [2023-04-29 09:45:09,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +41: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +10: [2023-04-29 09:45:09,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:09,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +10: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +29: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +21: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +21: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +21: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +10: [2023-04-29 09:45:09,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +45: [2023-04-29 09:45:09,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +21: [2023-04-29 09:45:09,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:09,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +29: [2023-04-29 09:45:09,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +27: [2023-04-29 09:45:09,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +26: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +58: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +19: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +29: [2023-04-29 09:45:09,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +19: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +19: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +13: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +19: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +17: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +17: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +43: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +20: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +20: [2023-04-29 09:45:09,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +20: [2023-04-29 09:45:09,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +28: [2023-04-29 09:45:09,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 7: [2023-04-29 09:45:09,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 6: [2023-04-29 09:45:09,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:09,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +26: [2023-04-29 09:45:09,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +17: [2023-04-29 09:45:09,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +17: [2023-04-29 09:45:09,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 4: [2023-04-29 09:45:09,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 4: [2023-04-29 09:45:09,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 6: [2023-04-29 09:45:09,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +51: [2023-04-29 09:45:09,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +39: [2023-04-29 09:45:09,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:09,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:09,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:09,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 6: [2023-04-29 09:45:09,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +32: [2023-04-29 09:45:09,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:09,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:09,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +27: [2023-04-29 09:45:09,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +27: [2023-04-29 09:45:09,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:09,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +21: [2023-04-29 09:45:09,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:09,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:09,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +14: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +51: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +21: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +43: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +21: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +21: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +51: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +27: [2023-04-29 09:45:09,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +30: [2023-04-29 09:45:09,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +30: [2023-04-29 09:45:09,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. + 5: [2023-04-29 09:45:09,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +49: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +49: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +49: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +49: [2023-04-29 09:45:09,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +17: [2023-04-29 09:45:09,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +16: [2023-04-29 09:45:09,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +27: [2023-04-29 09:45:09,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:09,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +30: [2023-04-29 09:45:09,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +17: [2023-04-29 09:45:09,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:09,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +32: [2023-04-29 09:45:09,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +51: [2023-04-29 09:45:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +43: [2023-04-29 09:45:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +43: [2023-04-29 09:45:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +17: [2023-04-29 09:45:09,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +13: [2023-04-29 09:45:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +38: [2023-04-29 09:45:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +38: [2023-04-29 09:45:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +38: [2023-04-29 09:45:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +38: [2023-04-29 09:45:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +17: [2023-04-29 09:45:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +13: [2023-04-29 09:45:09,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:09,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +20: [2023-04-29 09:45:09,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +28: [2023-04-29 09:45:09,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +32: [2023-04-29 09:45:09,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:09,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:09,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:09,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +25: [2023-04-29 09:45:09,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +27: [2023-04-29 09:45:09,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +27: [2023-04-29 09:45:09,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 8: [2023-04-29 09:45:09,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +25: [2023-04-29 09:45:09,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +49: [2023-04-29 09:45:09,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +49: [2023-04-29 09:45:09,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +32: [2023-04-29 09:45:09,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:09,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +28: [2023-04-29 09:45:09,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:09,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:09,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:09,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +27: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +49: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 3: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +58: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +58: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +30: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +58: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 2: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +51: [2023-04-29 09:45:09,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +49: [2023-04-29 09:45:09,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:09,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +30: [2023-04-29 09:45:09,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +29: [2023-04-29 09:45:09,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 2: [2023-04-29 09:45:09,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +41: [2023-04-29 09:45:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 3: [2023-04-29 09:45:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +41: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +34: [2023-04-29 09:45:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +34: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +34: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +38: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +27: [2023-04-29 09:45:09,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 2: [2023-04-29 09:45:09,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +10: [2023-04-29 09:45:09,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +10: [2023-04-29 09:45:09,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +38: [2023-04-29 09:45:09,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +32: [2023-04-29 09:45:09,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +10: [2023-04-29 09:45:09,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +28: [2023-04-29 09:45:09,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:09,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +38: [2023-04-29 09:45:09,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +13: [2023-04-29 09:45:09,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 3: [2023-04-29 09:45:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 4: [2023-04-29 09:45:09,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 4: [2023-04-29 09:45:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:09,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +29: [2023-04-29 09:45:09,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +43: [2023-04-29 09:45:09,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +28: [2023-04-29 09:45:09,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +43: [2023-04-29 09:45:09,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +29: [2023-04-29 09:45:09,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +13: [2023-04-29 09:45:09,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +13: [2023-04-29 09:45:09,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 8: [2023-04-29 09:45:09,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +13: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +31: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +28: [2023-04-29 09:45:09,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +14: [2023-04-29 09:45:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +29: [2023-04-29 09:45:09,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +29: [2023-04-29 09:45:09,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_10-model_01-model_states.pt. +37: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +37: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +28: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +37: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +28: [2023-04-29 09:45:09,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +28: [2023-04-29 09:45:09,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +37: [2023-04-29 09:45:09,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +28: [2023-04-29 09:45:09,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +19: [2023-04-29 09:45:09,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +19: [2023-04-29 09:45:09,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +37: [2023-04-29 09:45:09,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +37: [2023-04-29 09:45:09,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:09,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 8: [2023-04-29 09:45:09,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 8: [2023-04-29 09:45:09,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +14: [2023-04-29 09:45:09,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +13: [2023-04-29 09:45:09,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +13: [2023-04-29 09:45:09,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +39: [2023-04-29 09:45:09,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +25: [2023-04-29 09:45:09,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +61: [2023-04-29 09:45:09,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +61: [2023-04-29 09:45:09,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +61: [2023-04-29 09:45:09,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +61: [2023-04-29 09:45:09,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +25: [2023-04-29 09:45:09,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:09,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +25: [2023-04-29 09:45:09,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:09,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +10: [2023-04-29 09:45:09,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:09,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:09,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:09,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:09,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +17: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +34: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +28: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +28: [2023-04-29 09:45:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:09,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +17: [2023-04-29 09:45:09,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +23: [2023-04-29 09:45:09,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +23: [2023-04-29 09:45:09,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +23: [2023-04-29 09:45:09,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +10: [2023-04-29 09:45:09,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 5: [2023-04-29 09:45:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +10: [2023-04-29 09:45:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:09,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +17: [2023-04-29 09:45:09,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +59: [2023-04-29 09:45:09,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +59: [2023-04-29 09:45:09,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +59: [2023-04-29 09:45:09,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +59: [2023-04-29 09:45:09,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 0: [2023-04-29 09:45:09,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 0: [2023-04-29 09:45:09,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +14: [2023-04-29 09:45:09,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +14: [2023-04-29 09:45:09,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 4: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 5: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +17: [2023-04-29 09:45:09,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +31: [2023-04-29 09:45:09,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +56: [2023-04-29 09:45:09,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +56: [2023-04-29 09:45:09,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +17: [2023-04-29 09:45:09,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt... +39: [2023-04-29 09:45:09,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:09,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 0: [2023-04-29 09:45:09,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +14: [2023-04-29 09:45:09,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +56: [2023-04-29 09:45:09,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +56: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +17: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +19: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +39: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:09,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:09,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +17: [2023-04-29 09:45:09,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +61: [2023-04-29 09:45:09,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +19: [2023-04-29 09:45:09,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +61: [2023-04-29 09:45:09,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:09,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +56: [2023-04-29 09:45:09,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 7: [2023-04-29 09:45:09,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +19: [2023-04-29 09:45:09,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:09,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +19: [2023-04-29 09:45:09,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:09,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +34: [2023-04-29 09:45:09,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +23: [2023-04-29 09:45:09,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +34: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 1: [2023-04-29 09:45:09,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:09,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:09,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +23: [2023-04-29 09:45:09,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:09,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:09,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:09,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +28: [2023-04-29 09:45:09,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +38: [2023-04-29 09:45:09,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +30: [2023-04-29 09:45:09,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +30: [2023-04-29 09:45:09,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +23: [2023-04-29 09:45:09,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +34: [2023-04-29 09:45:09,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:09,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +34: [2023-04-29 09:45:09,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +59: [2023-04-29 09:45:09,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +59: [2023-04-29 09:45:09,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +59: [2023-04-29 09:45:09,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +15: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +15: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +15: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +30: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +15: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +38: [2023-04-29 09:45:09,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:09,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +38: [2023-04-29 09:45:09,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +38: [2023-04-29 09:45:09,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +56: [2023-04-29 09:45:09,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +56: [2023-04-29 09:45:09,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +56: [2023-04-29 09:45:09,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +56: [2023-04-29 09:45:09,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +38: [2023-04-29 09:45:09,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:09,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +63: [2023-04-29 09:45:09,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +38: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +16: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +16: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +16: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +16: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +63: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:09,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +63: [2023-04-29 09:45:09,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:09,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +42: [2023-04-29 09:45:09,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:09,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +58: [2023-04-29 09:45:09,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +34: [2023-04-29 09:45:09,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:09,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +34: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +34: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +34: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +15: [2023-04-29 09:45:09,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +50: [2023-04-29 09:45:09,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:09,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:09,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +15: [2023-04-29 09:45:09,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:09,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:09,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +58: [2023-04-29 09:45:09,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +58: [2023-04-29 09:45:09,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +31: [2023-04-29 09:45:09,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +56: [2023-04-29 09:45:09,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +56: [2023-04-29 09:45:09,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +58: [2023-04-29 09:45:09,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +31: [2023-04-29 09:45:09,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +56: [2023-04-29 09:45:09,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +56: [2023-04-29 09:45:09,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +31: [2023-04-29 09:45:09,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +60: [2023-04-29 09:45:09,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:09,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +31: [2023-04-29 09:45:09,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +56: [2023-04-29 09:45:09,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +31: [2023-04-29 09:45:09,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +31: [2023-04-29 09:45:09,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_00-model_states.pt. +33: [2023-04-29 09:45:09,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +33: [2023-04-29 09:45:09,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +33: [2023-04-29 09:45:09,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +60: [2023-04-29 09:45:09,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +60: [2023-04-29 09:45:09,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +34: [2023-04-29 09:45:09,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +31: [2023-04-29 09:45:09,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +31: [2023-04-29 09:45:09,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt... +33: [2023-04-29 09:45:09,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +37: [2023-04-29 09:45:09,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:09,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:09,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +56: [2023-04-29 09:45:09,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +56: [2023-04-29 09:45:09,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +35: [2023-04-29 09:45:09,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +35: [2023-04-29 09:45:09,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +34: [2023-04-29 09:45:09,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +36: [2023-04-29 09:45:09,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +36: [2023-04-29 09:45:09,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +35: [2023-04-29 09:45:09,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +36: [2023-04-29 09:45:09,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +36: [2023-04-29 09:45:09,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +56: [2023-04-29 09:45:09,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:09,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +38: [2023-04-29 09:45:09,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +35: [2023-04-29 09:45:09,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +36: [2023-04-29 09:45:09,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +38: [2023-04-29 09:45:09,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +35: [2023-04-29 09:45:09,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +38: [2023-04-29 09:45:09,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +56: [2023-04-29 09:45:09,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +38: [2023-04-29 09:45:09,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 7: [2023-04-29 09:45:09,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:09,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:09,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +50: [2023-04-29 09:45:09,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +56: [2023-04-29 09:45:09,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:09,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +42: [2023-04-29 09:45:09,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +42: [2023-04-29 09:45:09,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +42: [2023-04-29 09:45:09,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +42: [2023-04-29 09:45:09,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:09,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +34: [2023-04-29 09:45:09,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:09,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 1: [2023-04-29 09:45:09,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +50: [2023-04-29 09:45:09,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +24: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +24: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +57: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +24: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +57: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +57: [2023-04-29 09:45:09,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +57: [2023-04-29 09:45:09,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 1: [2023-04-29 09:45:09,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +38: [2023-04-29 09:45:09,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 1: [2023-04-29 09:45:09,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +62: [2023-04-29 09:45:09,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +62: [2023-04-29 09:45:09,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +62: [2023-04-29 09:45:09,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +38: [2023-04-29 09:45:09,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +62: [2023-04-29 09:45:09,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +24: [2023-04-29 09:45:09,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +48: [2023-04-29 09:45:09,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +48: [2023-04-29 09:45:09,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +63: [2023-04-29 09:45:09,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +24: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +14: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +14: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +34: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +24: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:09,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +48: [2023-04-29 09:45:09,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +38: [2023-04-29 09:45:09,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:09,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:09,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +34: [2023-04-29 09:45:09,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:09,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:09,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +56: [2023-04-29 09:45:09,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +38: [2023-04-29 09:45:09,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:09,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +14: [2023-04-29 09:45:09,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:09,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +14: [2023-04-29 09:45:09,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:09,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:09,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +14: [2023-04-29 09:45:09,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:09,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +42: [2023-04-29 09:45:09,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +33: [2023-04-29 09:45:09,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +63: [2023-04-29 09:45:09,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +63: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +63: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:09,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:09,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +16: [2023-04-29 09:45:09,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +49: [2023-04-29 09:45:09,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:09,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +35: [2023-04-29 09:45:09,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +35: [2023-04-29 09:45:09,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +35: [2023-04-29 09:45:09,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +50: [2023-04-29 09:45:09,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +49: [2023-04-29 09:45:09,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +33: [2023-04-29 09:45:09,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +49: [2023-04-29 09:45:09,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +50: [2023-04-29 09:45:09,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +49: [2023-04-29 09:45:09,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +57: [2023-04-29 09:45:09,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:09,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +16: [2023-04-29 09:45:09,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +57: [2023-04-29 09:45:09,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:09,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:09,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:09,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +24: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +42: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +16: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +42: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:09,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:09,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +49: [2023-04-29 09:45:09,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +49: [2023-04-29 09:45:09,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +49: [2023-04-29 09:45:09,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:09,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:09,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +24: [2023-04-29 09:45:09,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +32: [2023-04-29 09:45:09,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +32: [2023-04-29 09:45:09,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +16: [2023-04-29 09:45:09,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +62: [2023-04-29 09:45:09,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +32: [2023-04-29 09:45:09,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +56: [2023-04-29 09:45:09,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:09,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +48: [2023-04-29 09:45:09,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:09,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:09,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +62: [2023-04-29 09:45:09,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:09,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:09,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:09,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +46: [2023-04-29 09:45:09,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +46: [2023-04-29 09:45:09,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:09,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:09,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:09,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +56: [2023-04-29 09:45:09,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +22: [2023-04-29 09:45:09,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +22: [2023-04-29 09:45:09,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +56: [2023-04-29 09:45:09,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +46: [2023-04-29 09:45:09,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +46: [2023-04-29 09:45:09,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +36: [2023-04-29 09:45:09,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:09,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:09,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:09,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:09,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:09,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +46: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +22: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +46: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +33: [2023-04-29 09:45:10,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:10,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +54: [2023-04-29 09:45:10,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +60: [2023-04-29 09:45:10,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +33: [2023-04-29 09:45:10,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:10,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +24: [2023-04-29 09:45:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +42: [2023-04-29 09:45:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +35: [2023-04-29 09:45:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:10,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:10,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +33: [2023-04-29 09:45:10,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:10,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:10,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +44: [2023-04-29 09:45:10,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +44: [2023-04-29 09:45:10,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +32: [2023-04-29 09:45:10,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +35: [2023-04-29 09:45:10,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +44: [2023-04-29 09:45:10,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +44: [2023-04-29 09:45:10,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +44: [2023-04-29 09:45:10,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +32: [2023-04-29 09:45:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +44: [2023-04-29 09:45:10,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:10,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +22: [2023-04-29 09:45:10,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +12: [2023-04-29 09:45:10,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +12: [2023-04-29 09:45:10,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +12: [2023-04-29 09:45:10,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +22: [2023-04-29 09:45:10,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +35: [2023-04-29 09:45:10,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +22: [2023-04-29 09:45:10,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:10,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +36: [2023-04-29 09:45:10,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +43: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +43: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +43: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +20: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +49: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +20: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +24: [2023-04-29 09:45:10,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +33: [2023-04-29 09:45:10,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +41: [2023-04-29 09:45:10,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +41: [2023-04-29 09:45:10,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +41: [2023-04-29 09:45:10,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +11: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +11: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +41: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +11: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +33: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +33: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +11: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +33: [2023-04-29 09:45:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +29: [2023-04-29 09:45:10,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +20: [2023-04-29 09:45:10,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +20: [2023-04-29 09:45:10,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +42: [2023-04-29 09:45:10,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +20: [2023-04-29 09:45:10,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +29: [2023-04-29 09:45:10,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +20: [2023-04-29 09:45:10,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +35: [2023-04-29 09:45:10,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +20: [2023-04-29 09:45:10,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +14: [2023-04-29 09:45:10,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +14: [2023-04-29 09:45:10,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +14: [2023-04-29 09:45:10,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +42: [2023-04-29 09:45:10,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +20: [2023-04-29 09:45:10,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +20: [2023-04-29 09:45:10,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +35: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +14: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +59: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +40: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +40: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +59: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +40: [2023-04-29 09:45:10,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +12: [2023-04-29 09:45:10,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +35: [2023-04-29 09:45:10,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +45: [2023-04-29 09:45:10,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +12: [2023-04-29 09:45:10,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:10,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +45: [2023-04-29 09:45:10,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +18: [2023-04-29 09:45:10,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +18: [2023-04-29 09:45:10,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +57: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +57: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +24: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +24: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +57: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +12: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +49: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +12: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +59: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:10,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +18: [2023-04-29 09:45:10,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +49: [2023-04-29 09:45:10,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +49: [2023-04-29 09:45:10,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +42: [2023-04-29 09:45:10,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +49: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +42: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +42: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +52: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +52: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +52: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +59: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +52: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +24: [2023-04-29 09:45:10,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +43: [2023-04-29 09:45:10,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +59: [2023-04-29 09:45:10,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +59: [2023-04-29 09:45:10,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +59: [2023-04-29 09:45:10,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +24: [2023-04-29 09:45:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +41: [2023-04-29 09:45:10,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:10,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +42: [2023-04-29 09:45:10,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +57: [2023-04-29 09:45:10,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +41: [2023-04-29 09:45:10,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +43: [2023-04-29 09:45:10,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +11: [2023-04-29 09:45:10,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:10,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +41: [2023-04-29 09:45:10,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:10,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:10,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +46: [2023-04-29 09:45:10,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +44: [2023-04-29 09:45:10,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +11: [2023-04-29 09:45:10,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +41: [2023-04-29 09:45:10,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:10,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:10,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:10,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +33: [2023-04-29 09:45:10,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +57: [2023-04-29 09:45:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +10: [2023-04-29 09:45:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +10: [2023-04-29 09:45:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +10: [2023-04-29 09:45:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +10: [2023-04-29 09:45:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +24: [2023-04-29 09:45:10,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +14: [2023-04-29 09:45:10,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +24: [2023-04-29 09:45:10,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +40: [2023-04-29 09:45:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +40: [2023-04-29 09:45:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +14: [2023-04-29 09:45:10,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +14: [2023-04-29 09:45:10,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +44: [2023-04-29 09:45:10,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +45: [2023-04-29 09:45:10,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:10,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +44: [2023-04-29 09:45:10,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +40: [2023-04-29 09:45:10,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +40: [2023-04-29 09:45:10,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +40: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +40: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +40: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +19: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +19: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +35: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +40: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:10,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +39: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +24: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +44: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +39: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +26: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +18: [2023-04-29 09:45:10,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +26: [2023-04-29 09:45:10,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +26: [2023-04-29 09:45:10,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +52: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +19: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +40: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +19: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +40: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +52: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +49: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +24: [2023-04-29 09:45:10,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +52: [2023-04-29 09:45:10,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +24: [2023-04-29 09:45:10,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +24: [2023-04-29 09:45:10,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +49: [2023-04-29 09:45:10,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +35: [2023-04-29 09:45:10,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +54: [2023-04-29 09:45:10,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +40: [2023-04-29 09:45:10,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +18: [2023-04-29 09:45:10,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:10,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +52: [2023-04-29 09:45:10,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +49: [2023-04-29 09:45:10,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +37: [2023-04-29 09:45:10,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +37: [2023-04-29 09:45:10,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +37: [2023-04-29 09:45:10,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +46: [2023-04-29 09:45:10,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +35: [2023-04-29 09:45:10,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +18: [2023-04-29 09:45:10,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +46: [2023-04-29 09:45:10,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +24: [2023-04-29 09:45:10,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +35: [2023-04-29 09:45:10,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +44: [2023-04-29 09:45:10,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +10: [2023-04-29 09:45:10,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +46: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +10: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +47: [2023-04-29 09:45:10,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +47: [2023-04-29 09:45:10,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +47: [2023-04-29 09:45:10,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +57: [2023-04-29 09:45:10,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +54: [2023-04-29 09:45:10,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +35: [2023-04-29 09:45:10,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +20: [2023-04-29 09:45:10,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:10,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +61: [2023-04-29 09:45:10,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:10,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +10: [2023-04-29 09:45:10,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +61: [2023-04-29 09:45:10,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +61: [2023-04-29 09:45:10,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +26: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +61: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +61: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +10: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +27: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +27: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +27: [2023-04-29 09:45:10,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +27: [2023-04-29 09:45:10,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +61: [2023-04-29 09:45:10,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +61: [2023-04-29 09:45:10,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +26: [2023-04-29 09:45:10,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:10,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +57: [2023-04-29 09:45:10,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +39: [2023-04-29 09:45:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +22: [2023-04-29 09:45:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +22: [2023-04-29 09:45:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +26: [2023-04-29 09:45:10,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +44: [2023-04-29 09:45:10,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +29: [2023-04-29 09:45:10,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +29: [2023-04-29 09:45:10,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +29: [2023-04-29 09:45:10,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +44: [2023-04-29 09:45:10,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 3: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +44: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +22: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +39: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +22: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +22: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +22: [2023-04-29 09:45:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +22: [2023-04-29 09:45:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +29: [2023-04-29 09:45:10,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +22: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:10,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +57: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +19: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +19: [2023-04-29 09:45:10,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:10,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +24: [2023-04-29 09:45:10,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +37: [2023-04-29 09:45:10,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +20: [2023-04-29 09:45:10,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +37: [2023-04-29 09:45:10,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 8: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 8: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +41: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +13: [2023-04-29 09:45:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +59: [2023-04-29 09:45:10,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +13: [2023-04-29 09:45:10,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +13: [2023-04-29 09:45:10,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +47: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +46: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +54: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +24: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +37: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +20: [2023-04-29 09:45:10,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +46: [2023-04-29 09:45:10,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +41: [2023-04-29 09:45:10,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +47: [2023-04-29 09:45:10,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +47: [2023-04-29 09:45:10,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +47: [2023-04-29 09:45:10,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +54: [2023-04-29 09:45:10,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +59: [2023-04-29 09:45:10,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +23: [2023-04-29 09:45:10,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 6: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +23: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 6: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +23: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +17: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +41: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +33: [2023-04-29 09:45:10,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +20: [2023-04-29 09:45:10,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +59: [2023-04-29 09:45:10,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:10,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +41: [2023-04-29 09:45:10,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +23: [2023-04-29 09:45:10,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +23: [2023-04-29 09:45:10,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +23: [2023-04-29 09:45:10,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +17: [2023-04-29 09:45:10,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +23: [2023-04-29 09:45:10,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:10,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +33: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +17: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +23: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:10,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +41: [2023-04-29 09:45:10,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +23: [2023-04-29 09:45:10,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +41: [2023-04-29 09:45:10,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +14: [2023-04-29 09:45:10,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +14: [2023-04-29 09:45:10,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +23: [2023-04-29 09:45:10,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +23: [2023-04-29 09:45:10,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +59: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +25: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +33: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +33: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +25: [2023-04-29 09:45:10,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +33: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +29: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +55: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +55: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +55: [2023-04-29 09:45:10,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +14: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +55: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +41: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +15: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +45: [2023-04-29 09:45:10,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:10,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:10,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +25: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +15: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +15: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 2: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +45: [2023-04-29 09:45:10,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +30: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 1: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 1: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +25: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 1: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +15: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +58: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +58: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +30: [2023-04-29 09:45:10,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:10,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +29: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +33: [2023-04-29 09:45:10,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +33: [2023-04-29 09:45:10,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +20: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +25: [2023-04-29 09:45:10,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +11: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +46: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +33: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +13: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +46: [2023-04-29 09:45:10,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +13: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +13: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +25: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +59: [2023-04-29 09:45:10,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +20: [2023-04-29 09:45:10,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +45: [2023-04-29 09:45:10,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +29: [2023-04-29 09:45:10,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +45: [2023-04-29 09:45:10,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +46: [2023-04-29 09:45:10,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +11: [2023-04-29 09:45:10,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +40: [2023-04-29 09:45:10,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +11: [2023-04-29 09:45:10,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:10,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:10,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +11: [2023-04-29 09:45:10,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +11: [2023-04-29 09:45:10,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:10,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +59: [2023-04-29 09:45:10,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +11: [2023-04-29 09:45:10,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +28: [2023-04-29 09:45:10,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +28: [2023-04-29 09:45:10,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +28: [2023-04-29 09:45:10,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +28: [2023-04-29 09:45:10,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +59: [2023-04-29 09:45:10,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +40: [2023-04-29 09:45:10,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +20: [2023-04-29 09:45:10,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +40: [2023-04-29 09:45:10,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +39: [2023-04-29 09:45:10,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +61: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +14: [2023-04-29 09:45:10,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +39: [2023-04-29 09:45:10,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +12: [2023-04-29 09:45:10,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:10,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +40: [2023-04-29 09:45:10,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +40: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +39: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:10,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +26: [2023-04-29 09:45:10,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 5: [2023-04-29 09:45:10,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +55: [2023-04-29 09:45:10,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +55: [2023-04-29 09:45:10,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +40: [2023-04-29 09:45:10,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +14: [2023-04-29 09:45:10,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +61: [2023-04-29 09:45:10,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +61: [2023-04-29 09:45:10,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 1: [2023-04-29 09:45:10,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 1: [2023-04-29 09:45:10,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +55: [2023-04-29 09:45:10,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +55: [2023-04-29 09:45:10,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +63: [2023-04-29 09:45:10,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +63: [2023-04-29 09:45:10,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +63: [2023-04-29 09:45:10,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +63: [2023-04-29 09:45:10,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +39: [2023-04-29 09:45:10,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +39: [2023-04-29 09:45:10,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +26: [2023-04-29 09:45:10,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +22: [2023-04-29 09:45:10,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +58: [2023-04-29 09:45:10,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +22: [2023-04-29 09:45:10,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +58: [2023-04-29 09:45:10,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +26: [2023-04-29 09:45:10,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:10,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +39: [2023-04-29 09:45:10,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +40: [2023-04-29 09:45:10,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +22: [2023-04-29 09:45:10,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:10,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:10,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +40: [2023-04-29 09:45:10,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +26: [2023-04-29 09:45:10,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +45: [2023-04-29 09:45:10,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +61: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 0: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +28: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +45: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +28: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 0: [2023-04-29 09:45:10,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +28: [2023-04-29 09:45:10,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +28: [2023-04-29 09:45:10,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:10,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +51: [2023-04-29 09:45:10,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +40: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +23: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +38: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +38: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +38: [2023-04-29 09:45:10,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +38: [2023-04-29 09:45:10,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +15: [2023-04-29 09:45:10,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +63: [2023-04-29 09:45:10,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +17: [2023-04-29 09:45:10,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +31: [2023-04-29 09:45:10,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +63: [2023-04-29 09:45:10,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +22: [2023-04-29 09:45:10,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:10,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +63: [2023-04-29 09:45:10,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +63: [2023-04-29 09:45:10,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +23: [2023-04-29 09:45:10,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +29: [2023-04-29 09:45:10,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +29: [2023-04-29 09:45:10,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +26: [2023-04-29 09:45:10,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +25: [2023-04-29 09:45:10,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +17: [2023-04-29 09:45:10,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +17: [2023-04-29 09:45:10,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +29: [2023-04-29 09:45:10,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +22: [2023-04-29 09:45:10,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +29: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +29: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 3: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 3: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +26: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +15: [2023-04-29 09:45:10,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +25: [2023-04-29 09:45:10,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 3: [2023-04-29 09:45:10,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +46: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +46: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 3: [2023-04-29 09:45:10,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +11: [2023-04-29 09:45:10,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +15: [2023-04-29 09:45:10,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +30: [2023-04-29 09:45:10,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +14: [2023-04-29 09:45:10,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +46: [2023-04-29 09:45:10,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +25: [2023-04-29 09:45:10,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:10,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +34: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +34: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 2: [2023-04-29 09:45:10,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 6: [2023-04-29 09:45:10,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +34: [2023-04-29 09:45:10,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +34: [2023-04-29 09:45:10,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +11: [2023-04-29 09:45:10,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +46: [2023-04-29 09:45:10,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +15: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +14: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +25: [2023-04-29 09:45:10,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +38: [2023-04-29 09:45:10,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +23: [2023-04-29 09:45:10,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +48: [2023-04-29 09:45:10,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +17: [2023-04-29 09:45:10,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +38: [2023-04-29 09:45:10,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +31: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +31: [2023-04-29 09:45:10,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +31: [2023-04-29 09:45:10,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +17: [2023-04-29 09:45:10,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +23: [2023-04-29 09:45:10,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:10,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +31: [2023-04-29 09:45:10,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +48: [2023-04-29 09:45:10,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:10,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:10,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +25: [2023-04-29 09:45:10,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +48: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +21: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +14: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +14: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +15: [2023-04-29 09:45:10,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +12: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:10,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +30: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +48: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +17: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +21: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +53: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +53: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +21: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +48: [2023-04-29 09:45:10,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +30: [2023-04-29 09:45:10,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +53: [2023-04-29 09:45:10,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +53: [2023-04-29 09:45:10,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +21: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +21: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +21: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +36: [2023-04-29 09:45:10,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +36: [2023-04-29 09:45:10,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +36: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +36: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +21: [2023-04-29 09:45:10,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +21: [2023-04-29 09:45:10,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:10,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +47: [2023-04-29 09:45:10,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +47: [2023-04-29 09:45:10,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:10,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +15: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +47: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +11: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:10,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +14: [2023-04-29 09:45:10,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:10,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +45: [2023-04-29 09:45:10,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +18: [2023-04-29 09:45:10,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:10,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +11: [2023-04-29 09:45:10,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +34: [2023-04-29 09:45:10,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +18: [2023-04-29 09:45:10,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +18: [2023-04-29 09:45:10,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +18: [2023-04-29 09:45:10,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +34: [2023-04-29 09:45:10,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +34: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +14: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +55: [2023-04-29 09:45:10,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +55: [2023-04-29 09:45:10,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +17: [2023-04-29 09:45:10,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +45: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +17: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +25: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +60: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +60: [2023-04-29 09:45:10,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +60: [2023-04-29 09:45:10,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +12: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +27: [2023-04-29 09:45:10,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:10,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +56: [2023-04-29 09:45:10,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +56: [2023-04-29 09:45:10,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +56: [2023-04-29 09:45:10,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +27: [2023-04-29 09:45:10,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +27: [2023-04-29 09:45:10,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +27: [2023-04-29 09:45:10,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +51: [2023-04-29 09:45:10,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +16: [2023-04-29 09:45:10,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +16: [2023-04-29 09:45:10,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +16: [2023-04-29 09:45:10,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +16: [2023-04-29 09:45:10,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +27: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +27: [2023-04-29 09:45:10,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +52: [2023-04-29 09:45:10,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +27: [2023-04-29 09:45:10,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +52: [2023-04-29 09:45:10,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +51: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +51: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +52: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +51: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +52: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +52: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +52: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:10,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +52: [2023-04-29 09:45:10,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +52: [2023-04-29 09:45:10,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +36: [2023-04-29 09:45:10,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +53: [2023-04-29 09:45:10,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +53: [2023-04-29 09:45:10,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +36: [2023-04-29 09:45:10,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +13: [2023-04-29 09:45:10,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +60: [2023-04-29 09:45:10,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +60: [2023-04-29 09:45:10,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:10,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +30: [2023-04-29 09:45:10,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +13: [2023-04-29 09:45:10,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:10,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +13: [2023-04-29 09:45:10,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +45: [2023-04-29 09:45:10,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +56: [2023-04-29 09:45:10,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +13: [2023-04-29 09:45:10,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +56: [2023-04-29 09:45:10,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +13: [2023-04-29 09:45:10,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +13: [2023-04-29 09:45:10,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +13: [2023-04-29 09:45:10,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +30: [2023-04-29 09:45:10,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_11-model_01-model_states.pt. +13: [2023-04-29 09:45:10,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +56: [2023-04-29 09:45:10,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:10,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +60: [2023-04-29 09:45:10,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:10,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +16: [2023-04-29 09:45:10,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:10,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +16: [2023-04-29 09:45:10,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +16: [2023-04-29 09:45:10,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +45: [2023-04-29 09:45:10,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:10,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:10,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:10,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +48: [2023-04-29 09:45:10,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +51: [2023-04-29 09:45:10,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +18: [2023-04-29 09:45:10,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +49: [2023-04-29 09:45:10,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +49: [2023-04-29 09:45:10,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +49: [2023-04-29 09:45:10,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +49: [2023-04-29 09:45:10,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +47: [2023-04-29 09:45:10,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +17: [2023-04-29 09:45:10,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:10,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +58: [2023-04-29 09:45:10,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +58: [2023-04-29 09:45:10,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +58: [2023-04-29 09:45:10,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +58: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +17: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +21: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +59: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +59: [2023-04-29 09:45:10,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +17: [2023-04-29 09:45:10,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 5: [2023-04-29 09:45:10,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:10,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +59: [2023-04-29 09:45:10,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +30: [2023-04-29 09:45:10,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 5: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +30: [2023-04-29 09:45:10,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +47: [2023-04-29 09:45:10,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +27: [2023-04-29 09:45:10,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +48: [2023-04-29 09:45:10,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +18: [2023-04-29 09:45:10,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +18: [2023-04-29 09:45:10,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +18: [2023-04-29 09:45:10,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +55: [2023-04-29 09:45:10,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +55: [2023-04-29 09:45:10,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +27: [2023-04-29 09:45:10,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +54: [2023-04-29 09:45:10,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +54: [2023-04-29 09:45:10,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +54: [2023-04-29 09:45:10,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +18: [2023-04-29 09:45:10,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. + 3: [2023-04-29 09:45:10,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +31: [2023-04-29 09:45:10,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +31: [2023-04-29 09:45:10,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +48: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +19: [2023-04-29 09:45:10,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +47: [2023-04-29 09:45:10,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +50: [2023-04-29 09:45:10,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +19: [2023-04-29 09:45:10,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +17: [2023-04-29 09:45:10,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +50: [2023-04-29 09:45:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +52: [2023-04-29 09:45:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +19: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +27: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +37: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +19: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +19: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +19: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +37: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +19: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +31: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +37: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +31: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +31: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +31: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +42: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +42: [2023-04-29 09:45:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +31: [2023-04-29 09:45:10,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +19: [2023-04-29 09:45:10,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +31: [2023-04-29 09:45:10,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +21: [2023-04-29 09:45:10,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +49: [2023-04-29 09:45:10,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +21: [2023-04-29 09:45:10,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:10,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +42: [2023-04-29 09:45:10,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +21: [2023-04-29 09:45:10,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +49: [2023-04-29 09:45:10,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +21: [2023-04-29 09:45:10,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +59: [2023-04-29 09:45:10,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +49: [2023-04-29 09:45:10,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +59: [2023-04-29 09:45:10,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +49: [2023-04-29 09:45:10,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +37: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +37: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +53: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +59: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +27: [2023-04-29 09:45:10,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +37: [2023-04-29 09:45:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +53: [2023-04-29 09:45:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +47: [2023-04-29 09:45:10,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +47: [2023-04-29 09:45:10,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +47: [2023-04-29 09:45:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +53: [2023-04-29 09:45:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +19: [2023-04-29 09:45:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +18: [2023-04-29 09:45:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +59: [2023-04-29 09:45:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +19: [2023-04-29 09:45:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +53: [2023-04-29 09:45:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +19: [2023-04-29 09:45:10,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt... +60: [2023-04-29 09:45:10,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +52: [2023-04-29 09:45:10,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:10,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +41: [2023-04-29 09:45:10,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +41: [2023-04-29 09:45:10,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +53: [2023-04-29 09:45:10,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +18: [2023-04-29 09:45:10,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +18: [2023-04-29 09:45:10,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +44: [2023-04-29 09:45:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +44: [2023-04-29 09:45:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +41: [2023-04-29 09:45:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +13: [2023-04-29 09:45:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +44: [2023-04-29 09:45:10,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +27: [2023-04-29 09:45:10,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +44: [2023-04-29 09:45:10,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_32-model_01-model_states.pt. +60: [2023-04-29 09:45:10,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +60: [2023-04-29 09:45:10,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +55: [2023-04-29 09:45:10,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +55: [2023-04-29 09:45:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +55: [2023-04-29 09:45:10,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +55: [2023-04-29 09:45:10,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +57: [2023-04-29 09:45:10,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +57: [2023-04-29 09:45:10,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +57: [2023-04-29 09:45:10,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +52: [2023-04-29 09:45:10,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +60: [2023-04-29 09:45:10,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +60: [2023-04-29 09:45:10,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +39: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:10,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:10,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:10,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +50: [2023-04-29 09:45:10,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +39: [2023-04-29 09:45:10,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:10,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:10,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:10,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +30: [2023-04-29 09:45:10,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +35: [2023-04-29 09:45:10,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +35: [2023-04-29 09:45:10,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +35: [2023-04-29 09:45:10,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +60: [2023-04-29 09:45:10,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +60: [2023-04-29 09:45:10,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +35: [2023-04-29 09:45:10,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +51: [2023-04-29 09:45:10,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +51: [2023-04-29 09:45:10,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +58: [2023-04-29 09:45:10,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +30: [2023-04-29 09:45:10,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +51: [2023-04-29 09:45:10,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +58: [2023-04-29 09:45:10,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +13: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +41: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +13: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +13: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +41: [2023-04-29 09:45:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +13: [2023-04-29 09:45:10,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +44: [2023-04-29 09:45:10,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:10,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +44: [2023-04-29 09:45:10,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:10,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +44: [2023-04-29 09:45:10,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +41: [2023-04-29 09:45:10,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +30: [2023-04-29 09:45:10,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +57: [2023-04-29 09:45:10,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +31: [2023-04-29 09:45:10,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +45: [2023-04-29 09:45:10,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +45: [2023-04-29 09:45:10,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +45: [2023-04-29 09:45:10,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +45: [2023-04-29 09:45:10,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +30: [2023-04-29 09:45:10,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +30: [2023-04-29 09:45:10,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt... +31: [2023-04-29 09:45:10,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +35: [2023-04-29 09:45:10,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:10,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +31: [2023-04-29 09:45:10,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +58: [2023-04-29 09:45:10,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:10,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +35: [2023-04-29 09:45:10,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +58: [2023-04-29 09:45:10,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:10,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +31: [2023-04-29 09:45:10,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +45: [2023-04-29 09:45:10,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +53: [2023-04-29 09:45:10,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +53: [2023-04-29 09:45:10,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +53: [2023-04-29 09:45:10,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +31: [2023-04-29 09:45:10,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +50: [2023-04-29 09:45:10,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +53: [2023-04-29 09:45:10,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +19: [2023-04-29 09:45:10,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +19: [2023-04-29 09:45:10,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_00-model_states.pt. +53: [2023-04-29 09:45:10,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +53: [2023-04-29 09:45:10,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +31: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +19: [2023-04-29 09:45:10,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +31: [2023-04-29 09:45:10,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +43: [2023-04-29 09:45:10,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +61: [2023-04-29 09:45:10,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +61: [2023-04-29 09:45:10,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +61: [2023-04-29 09:45:10,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +61: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +41: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:10,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +43: [2023-04-29 09:45:10,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +43: [2023-04-29 09:45:10,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +43: [2023-04-29 09:45:10,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +43: [2023-04-29 09:45:10,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +41: [2023-04-29 09:45:10,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +37: [2023-04-29 09:45:10,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:10,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:10,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +44: [2023-04-29 09:45:10,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +50: [2023-04-29 09:45:10,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +50: [2023-04-29 09:45:10,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +41: [2023-04-29 09:45:10,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +24: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +24: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +24: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +24: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +44: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +11: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +11: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +39: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +39: [2023-04-29 09:45:10,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +39: [2023-04-29 09:45:10,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +39: [2023-04-29 09:45:10,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +44: [2023-04-29 09:45:10,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +41: [2023-04-29 09:45:10,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +44: [2023-04-29 09:45:10,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +11: [2023-04-29 09:45:10,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +19: [2023-04-29 09:45:10,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +38: [2023-04-29 09:45:10,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +38: [2023-04-29 09:45:10,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +38: [2023-04-29 09:45:10,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +38: [2023-04-29 09:45:10,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +38: [2023-04-29 09:45:10,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +38: [2023-04-29 09:45:10,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +20: [2023-04-29 09:45:10,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +20: [2023-04-29 09:45:10,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +22: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +22: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +41: [2023-04-29 09:45:10,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +19: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +41: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +20: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +20: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +39: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +19: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +34: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +22: [2023-04-29 09:45:10,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +22: [2023-04-29 09:45:10,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +41: [2023-04-29 09:45:10,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +42: [2023-04-29 09:45:10,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +61: [2023-04-29 09:45:10,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +42: [2023-04-29 09:45:10,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +41: [2023-04-29 09:45:10,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +42: [2023-04-29 09:45:10,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +24: [2023-04-29 09:45:10,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +42: [2023-04-29 09:45:10,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +39: [2023-04-29 09:45:10,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +42: [2023-04-29 09:45:10,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +32: [2023-04-29 09:45:10,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +32: [2023-04-29 09:45:10,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +32: [2023-04-29 09:45:10,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +33: [2023-04-29 09:45:10,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +33: [2023-04-29 09:45:10,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +39: [2023-04-29 09:45:10,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:10,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +24: [2023-04-29 09:45:10,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +45: [2023-04-29 09:45:10,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +33: [2023-04-29 09:45:10,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +39: [2023-04-29 09:45:10,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +24: [2023-04-29 09:45:10,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +45: [2023-04-29 09:45:10,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 4: [2023-04-29 09:45:10,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +40: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +40: [2023-04-29 09:45:10,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +40: [2023-04-29 09:45:10,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +62: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +62: [2023-04-29 09:45:10,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +20: [2023-04-29 09:45:10,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +45: [2023-04-29 09:45:10,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:10,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +20: [2023-04-29 09:45:10,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +46: [2023-04-29 09:45:10,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +46: [2023-04-29 09:45:10,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +46: [2023-04-29 09:45:10,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +46: [2023-04-29 09:45:10,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +62: [2023-04-29 09:45:10,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +62: [2023-04-29 09:45:10,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +23: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +23: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +53: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +23: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +45: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +43: [2023-04-29 09:45:10,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +62: [2023-04-29 09:45:10,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +23: [2023-04-29 09:45:10,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +25: [2023-04-29 09:45:10,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +25: [2023-04-29 09:45:10,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +62: [2023-04-29 09:45:10,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt... +53: [2023-04-29 09:45:10,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt... +25: [2023-04-29 09:45:10,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +25: [2023-04-29 09:45:10,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +41: [2023-04-29 09:45:10,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +53: [2023-04-29 09:45:10,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +36: [2023-04-29 09:45:10,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +36: [2023-04-29 09:45:10,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +43: [2023-04-29 09:45:10,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +36: [2023-04-29 09:45:10,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +36: [2023-04-29 09:45:10,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +33: [2023-04-29 09:45:10,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:10,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +40: [2023-04-29 09:45:10,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +40: [2023-04-29 09:45:10,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +23: [2023-04-29 09:45:10,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +23: [2023-04-29 09:45:10,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +55: [2023-04-29 09:45:10,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +55: [2023-04-29 09:45:10,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +55: [2023-04-29 09:45:10,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +55: [2023-04-29 09:45:10,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +38: [2023-04-29 09:45:10,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +23: [2023-04-29 09:45:10,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +42: [2023-04-29 09:45:10,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +25: [2023-04-29 09:45:10,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +41: [2023-04-29 09:45:10,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +25: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +47: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +47: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +52: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +47: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. + 9: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +52: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +52: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +47: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +25: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +52: [2023-04-29 09:45:10,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +42: [2023-04-29 09:45:10,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +12: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +12: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +34: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +34: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +12: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +41: [2023-04-29 09:45:10,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +25: [2023-04-29 09:45:10,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +39: [2023-04-29 09:45:10,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +32: [2023-04-29 09:45:10,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +62: [2023-04-29 09:45:10,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +38: [2023-04-29 09:45:10,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +39: [2023-04-29 09:45:10,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:10,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +43: [2023-04-29 09:45:10,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +41: [2023-04-29 09:45:10,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 1: [2023-04-29 09:45:10,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +38: [2023-04-29 09:45:10,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 1: [2023-04-29 09:45:10,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +39: [2023-04-29 09:45:10,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +39: [2023-04-29 09:45:10,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:10,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +47: [2023-04-29 09:45:10,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +55: [2023-04-29 09:45:10,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +55: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +38: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +60: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +36: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:10,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +12: [2023-04-29 09:45:10,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +47: [2023-04-29 09:45:10,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +47: [2023-04-29 09:45:10,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +60: [2023-04-29 09:45:10,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +62: [2023-04-29 09:45:10,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +12: [2023-04-29 09:45:10,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +42: [2023-04-29 09:45:10,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +53: [2023-04-29 09:45:10,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +53: [2023-04-29 09:45:10,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +12: [2023-04-29 09:45:10,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +12: [2023-04-29 09:45:10,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +52: [2023-04-29 09:45:10,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:10,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:10,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:10,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:10,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:10,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +28: [2023-04-29 09:45:10,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +32: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +28: [2023-04-29 09:45:10,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +32: [2023-04-29 09:45:10,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +62: [2023-04-29 09:45:10,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:10,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +48: [2023-04-29 09:45:10,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +28: [2023-04-29 09:45:10,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +36: [2023-04-29 09:45:10,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +28: [2023-04-29 09:45:10,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +48: [2023-04-29 09:45:10,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +48: [2023-04-29 09:45:10,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +48: [2023-04-29 09:45:10,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +62: [2023-04-29 09:45:10,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_00-model_states.pt. +48: [2023-04-29 09:45:10,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:10,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +28: [2023-04-29 09:45:10,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +28: [2023-04-29 09:45:10,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +28: [2023-04-29 09:45:10,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +28: [2023-04-29 09:45:10,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +28: [2023-04-29 09:45:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +28: [2023-04-29 09:45:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +60: [2023-04-29 09:45:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +60: [2023-04-29 09:45:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:10,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:10,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +60: [2023-04-29 09:45:10,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:10,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:10,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +15: [2023-04-29 09:45:10,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +15: [2023-04-29 09:45:10,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +15: [2023-04-29 09:45:10,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +15: [2023-04-29 09:45:10,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +13: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +13: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +13: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +53: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +13: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:10,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +16: [2023-04-29 09:45:10,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +16: [2023-04-29 09:45:10,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +16: [2023-04-29 09:45:10,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +16: [2023-04-29 09:45:10,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +51: [2023-04-29 09:45:10,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +51: [2023-04-29 09:45:10,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +16: [2023-04-29 09:45:10,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +16: [2023-04-29 09:45:10,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +62: [2023-04-29 09:45:10,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +36: [2023-04-29 09:45:10,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:10,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +51: [2023-04-29 09:45:10,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +16: [2023-04-29 09:45:10,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +62: [2023-04-29 09:45:10,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +10: [2023-04-29 09:45:10,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +48: [2023-04-29 09:45:10,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:10,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +10: [2023-04-29 09:45:10,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +10: [2023-04-29 09:45:10,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +10: [2023-04-29 09:45:10,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +15: [2023-04-29 09:45:10,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +13: [2023-04-29 09:45:10,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +15: [2023-04-29 09:45:10,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +15: [2023-04-29 09:45:10,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +13: [2023-04-29 09:45:10,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +15: [2023-04-29 09:45:10,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +13: [2023-04-29 09:45:10,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +13: [2023-04-29 09:45:10,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 3: [2023-04-29 09:45:10,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 3: [2023-04-29 09:45:10,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 3: [2023-04-29 09:45:10,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 3: [2023-04-29 09:45:10,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +51: [2023-04-29 09:45:10,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +26: [2023-04-29 09:45:10,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +26: [2023-04-29 09:45:10,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +26: [2023-04-29 09:45:10,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +26: [2023-04-29 09:45:10,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 1: [2023-04-29 09:45:10,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:10,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +14: [2023-04-29 09:45:10,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +14: [2023-04-29 09:45:10,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +14: [2023-04-29 09:45:10,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +28: [2023-04-29 09:45:10,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:10,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +44: [2023-04-29 09:45:10,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +44: [2023-04-29 09:45:10,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +48: [2023-04-29 09:45:10,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +50: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +50: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +50: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +44: [2023-04-29 09:45:10,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +44: [2023-04-29 09:45:10,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +27: [2023-04-29 09:45:10,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +27: [2023-04-29 09:45:10,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +51: [2023-04-29 09:45:10,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:10,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +29: [2023-04-29 09:45:10,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 2: [2023-04-29 09:45:10,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 1: [2023-04-29 09:45:10,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +29: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +24: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +14: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 8: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 8: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +14: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +29: [2023-04-29 09:45:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +53: [2023-04-29 09:45:10,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +29: [2023-04-29 09:45:10,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +14: [2023-04-29 09:45:10,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +24: [2023-04-29 09:45:10,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 2: [2023-04-29 09:45:10,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +24: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +48: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +24: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +24: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +24: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +29: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +20: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +28: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +28: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +56: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +14: [2023-04-29 09:45:10,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +18: [2023-04-29 09:45:10,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +14: [2023-04-29 09:45:10,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +60: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +14: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +60: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +60: [2023-04-29 09:45:10,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +58: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +58: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +60: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +18: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +58: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +29: [2023-04-29 09:45:10,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +56: [2023-04-29 09:45:10,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:10,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +56: [2023-04-29 09:45:10,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +56: [2023-04-29 09:45:10,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +26: [2023-04-29 09:45:10,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +53: [2023-04-29 09:45:10,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +28: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +56: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +48: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +60: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +14: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:10,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 2: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +60: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +28: [2023-04-29 09:45:10,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +56: [2023-04-29 09:45:10,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +48: [2023-04-29 09:45:10,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +60: [2023-04-29 09:45:10,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +26: [2023-04-29 09:45:10,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +16: [2023-04-29 09:45:10,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +60: [2023-04-29 09:45:10,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +44: [2023-04-29 09:45:10,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +53: [2023-04-29 09:45:10,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 6: [2023-04-29 09:45:10,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +44: [2023-04-29 09:45:10,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +45: [2023-04-29 09:45:10,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:10,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:10,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +27: [2023-04-29 09:45:10,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +44: [2023-04-29 09:45:10,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 6: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +37: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +60: [2023-04-29 09:45:10,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +37: [2023-04-29 09:45:10,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +50: [2023-04-29 09:45:10,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. + 6: [2023-04-29 09:45:10,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +50: [2023-04-29 09:45:10,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +14: [2023-04-29 09:45:10,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +53: [2023-04-29 09:45:10,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 0: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +27: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +29: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +29: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +14: [2023-04-29 09:45:10,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +14: [2023-04-29 09:45:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +50: [2023-04-29 09:45:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +50: [2023-04-29 09:45:10,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:10,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +27: [2023-04-29 09:45:10,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +50: [2023-04-29 09:45:10,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +33: [2023-04-29 09:45:10,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +33: [2023-04-29 09:45:10,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +50: [2023-04-29 09:45:10,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +16: [2023-04-29 09:45:10,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +33: [2023-04-29 09:45:10,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +58: [2023-04-29 09:45:10,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +21: [2023-04-29 09:45:10,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +21: [2023-04-29 09:45:10,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +21: [2023-04-29 09:45:10,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +48: [2023-04-29 09:45:10,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +21: [2023-04-29 09:45:10,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +33: [2023-04-29 09:45:10,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +33: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +33: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +33: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +48: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +18: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +19: [2023-04-29 09:45:10,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +18: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +28: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 7: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +58: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +50: [2023-04-29 09:45:10,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +50: [2023-04-29 09:45:10,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:10,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +16: [2023-04-29 09:45:10,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +48: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +48: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +58: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +58: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +58: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +62: [2023-04-29 09:45:10,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +10: [2023-04-29 09:45:10,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:10,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:10,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +62: [2023-04-29 09:45:10,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_33-model_01-model_states.pt. +10: [2023-04-29 09:45:10,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +37: [2023-04-29 09:45:10,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:10,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:10,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +10: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +48: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +17: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +48: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +17: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +58: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +24: [2023-04-29 09:45:10,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +17: [2023-04-29 09:45:10,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +45: [2023-04-29 09:45:10,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +45: [2023-04-29 09:45:10,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +51: [2023-04-29 09:45:10,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +41: [2023-04-29 09:45:10,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +41: [2023-04-29 09:45:10,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +41: [2023-04-29 09:45:10,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +41: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +51: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +51: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +26: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +57: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +57: [2023-04-29 09:45:10,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +48: [2023-04-29 09:45:10,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +20: [2023-04-29 09:45:10,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +48: [2023-04-29 09:45:10,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +10: [2023-04-29 09:45:10,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +48: [2023-04-29 09:45:10,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +31: [2023-04-29 09:45:10,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +31: [2023-04-29 09:45:10,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +26: [2023-04-29 09:45:10,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +51: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +23: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +23: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +51: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +26: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +57: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +21: [2023-04-29 09:45:10,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +57: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +57: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +57: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +57: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +51: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +16: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:10,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +19: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +14: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +43: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +43: [2023-04-29 09:45:10,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +30: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +30: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +17: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +22: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +22: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +22: [2023-04-29 09:45:10,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +23: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +22: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +51: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +59: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +43: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +17: [2023-04-29 09:45:10,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +16: [2023-04-29 09:45:10,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:10,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +30: [2023-04-29 09:45:10,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +30: [2023-04-29 09:45:10,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +22: [2023-04-29 09:45:10,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +51: [2023-04-29 09:45:10,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +22: [2023-04-29 09:45:10,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +24: [2023-04-29 09:45:10,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:10,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:10,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +17: [2023-04-29 09:45:10,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +56: [2023-04-29 09:45:10,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:10,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +20: [2023-04-29 09:45:10,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +24: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +26: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +54: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +60: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +10: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +54: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +17: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +17: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +17: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +10: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +62: [2023-04-29 09:45:10,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +59: [2023-04-29 09:45:10,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +59: [2023-04-29 09:45:10,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +50: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +17: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +29: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +12: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +17: [2023-04-29 09:45:10,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +14: [2023-04-29 09:45:10,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +56: [2023-04-29 09:45:10,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +20: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +59: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +59: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +59: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 2: [2023-04-29 09:45:10,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +60: [2023-04-29 09:45:10,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +14: [2023-04-29 09:45:10,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +60: [2023-04-29 09:45:10,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +17: [2023-04-29 09:45:10,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +41: [2023-04-29 09:45:10,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +54: [2023-04-29 09:45:10,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +12: [2023-04-29 09:45:10,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +41: [2023-04-29 09:45:10,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +54: [2023-04-29 09:45:10,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +56: [2023-04-29 09:45:10,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +12: [2023-04-29 09:45:10,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +31: [2023-04-29 09:45:10,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +26: [2023-04-29 09:45:10,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +31: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +12: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +12: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +26: [2023-04-29 09:45:10,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +33: [2023-04-29 09:45:10,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +14: [2023-04-29 09:45:10,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +50: [2023-04-29 09:45:10,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +26: [2023-04-29 09:45:10,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +43: [2023-04-29 09:45:10,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +26: [2023-04-29 09:45:10,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +33: [2023-04-29 09:45:10,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +33: [2023-04-29 09:45:10,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +30: [2023-04-29 09:45:10,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +56: [2023-04-29 09:45:10,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +29: [2023-04-29 09:45:10,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +56: [2023-04-29 09:45:10,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +29: [2023-04-29 09:45:10,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +60: [2023-04-29 09:45:10,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +29: [2023-04-29 09:45:10,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +58: [2023-04-29 09:45:10,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +29: [2023-04-29 09:45:10,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +42: [2023-04-29 09:45:10,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +42: [2023-04-29 09:45:10,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +42: [2023-04-29 09:45:10,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +42: [2023-04-29 09:45:10,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +24: [2023-04-29 09:45:10,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:10,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:10,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +24: [2023-04-29 09:45:10,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:10,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +29: [2023-04-29 09:45:10,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 2: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +58: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +14: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +60: [2023-04-29 09:45:10,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +29: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 5: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +20: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. + 5: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +20: [2023-04-29 09:45:10,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +20: [2023-04-29 09:45:10,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +29: [2023-04-29 09:45:10,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +56: [2023-04-29 09:45:10,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:10,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +33: [2023-04-29 09:45:10,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +11: [2023-04-29 09:45:10,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +11: [2023-04-29 09:45:10,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +15: [2023-04-29 09:45:10,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +15: [2023-04-29 09:45:10,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +15: [2023-04-29 09:45:10,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +51: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +15: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +11: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +11: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +60: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +11: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 2: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +11: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +60: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +11: [2023-04-29 09:45:10,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +50: [2023-04-29 09:45:10,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +14: [2023-04-29 09:45:10,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +56: [2023-04-29 09:45:10,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +32: [2023-04-29 09:45:10,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +58: [2023-04-29 09:45:10,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:10,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +58: [2023-04-29 09:45:10,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:10,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +32: [2023-04-29 09:45:10,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +29: [2023-04-29 09:45:10,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:10,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:10,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +11: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +50: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +26: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 6: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +50: [2023-04-29 09:45:10,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 6: [2023-04-29 09:45:10,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:10,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +11: [2023-04-29 09:45:10,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +11: [2023-04-29 09:45:10,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:10,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:10,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +29: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +58: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +50: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +58: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:10,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:10,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +14: [2023-04-29 09:45:10,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +50: [2023-04-29 09:45:10,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +57: [2023-04-29 09:45:10,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +57: [2023-04-29 09:45:10,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +50: [2023-04-29 09:45:10,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +63: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +42: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +30: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +63: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +22: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +51: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +63: [2023-04-29 09:45:10,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +63: [2023-04-29 09:45:10,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +63: [2023-04-29 09:45:10,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +57: [2023-04-29 09:45:10,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +26: [2023-04-29 09:45:10,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_12-model_01-model_states.pt. +33: [2023-04-29 09:45:10,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:10,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +42: [2023-04-29 09:45:10,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:10,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +29: [2023-04-29 09:45:10,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +23: [2023-04-29 09:45:10,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +57: [2023-04-29 09:45:10,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +26: [2023-04-29 09:45:10,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +29: [2023-04-29 09:45:10,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +30: [2023-04-29 09:45:10,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +23: [2023-04-29 09:45:10,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:10,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:10,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +14: [2023-04-29 09:45:10,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +22: [2023-04-29 09:45:10,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +14: [2023-04-29 09:45:10,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:10,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:10,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +58: [2023-04-29 09:45:10,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +17: [2023-04-29 09:45:10,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +58: [2023-04-29 09:45:10,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:10,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:10,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +38: [2023-04-29 09:45:10,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +17: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +32: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +12: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +38: [2023-04-29 09:45:10,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +54: [2023-04-29 09:45:10,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:10,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +58: [2023-04-29 09:45:10,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:10,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +59: [2023-04-29 09:45:10,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +58: [2023-04-29 09:45:10,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +45: [2023-04-29 09:45:10,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:10,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +17: [2023-04-29 09:45:10,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +26: [2023-04-29 09:45:10,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +45: [2023-04-29 09:45:10,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +57: [2023-04-29 09:45:10,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +57: [2023-04-29 09:45:10,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 3: [2023-04-29 09:45:10,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +17: [2023-04-29 09:45:10,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +22: [2023-04-29 09:45:10,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +12: [2023-04-29 09:45:10,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:10,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:10,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +51: [2023-04-29 09:45:10,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:10,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +17: [2023-04-29 09:45:10,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +50: [2023-04-29 09:45:10,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:10,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +30: [2023-04-29 09:45:10,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +57: [2023-04-29 09:45:10,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 3: [2023-04-29 09:45:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +54: [2023-04-29 09:45:10,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +15: [2023-04-29 09:45:10,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +57: [2023-04-29 09:45:10,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +26: [2023-04-29 09:45:10,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +23: [2023-04-29 09:45:10,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +23: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +23: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +22: [2023-04-29 09:45:10,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +22: [2023-04-29 09:45:10,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +26: [2023-04-29 09:45:10,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:10,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +49: [2023-04-29 09:45:10,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +49: [2023-04-29 09:45:10,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +49: [2023-04-29 09:45:10,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +13: [2023-04-29 09:45:10,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +11: [2023-04-29 09:45:10,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +17: [2023-04-29 09:45:10,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +22: [2023-04-29 09:45:10,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +13: [2023-04-29 09:45:10,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +49: [2023-04-29 09:45:10,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +13: [2023-04-29 09:45:10,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +34: [2023-04-29 09:45:10,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +34: [2023-04-29 09:45:10,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +13: [2023-04-29 09:45:10,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +13: [2023-04-29 09:45:10,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +13: [2023-04-29 09:45:10,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +34: [2023-04-29 09:45:10,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +34: [2023-04-29 09:45:10,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +59: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +30: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +30: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +13: [2023-04-29 09:45:10,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +12: [2023-04-29 09:45:10,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +38: [2023-04-29 09:45:10,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +12: [2023-04-29 09:45:10,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +59: [2023-04-29 09:45:10,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +27: [2023-04-29 09:45:10,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +38: [2023-04-29 09:45:10,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +27: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +54: [2023-04-29 09:45:10,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +12: [2023-04-29 09:45:10,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +40: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +40: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +38: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +38: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +17: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +17: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +11: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +17: [2023-04-29 09:45:10,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:10,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +59: [2023-04-29 09:45:10,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +30: [2023-04-29 09:45:10,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +30: [2023-04-29 09:45:10,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +40: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +40: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +30: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +40: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +40: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +40: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +27: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +25: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +40: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +18: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +25: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +12: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +21: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +27: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +61: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:10,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +15: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +39: [2023-04-29 09:45:10,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:10,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:10,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +27: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +27: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +61: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +11: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +61: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +18: [2023-04-29 09:45:10,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 7: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +40: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +40: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +40: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +40: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +40: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +25: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:10,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 0: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +40: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +25: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +40: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +18: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +21: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +54: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +21: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +21: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +21: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +21: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +39: [2023-04-29 09:45:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +21: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +18: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +37: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +46: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +21: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +61: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +61: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... + 9: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +37: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +44: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +61: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +37: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +53: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +18: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +53: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +25: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +18: [2023-04-29 09:45:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +61: [2023-04-29 09:45:10,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +18: [2023-04-29 09:45:10,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +18: [2023-04-29 09:45:10,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +53: [2023-04-29 09:45:10,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +25: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +25: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +25: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +46: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 7: [2023-04-29 09:45:10,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +46: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +36: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +36: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +54: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +25: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +36: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +11: [2023-04-29 09:45:10,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +63: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +63: [2023-04-29 09:45:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 7: [2023-04-29 09:45:10,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +63: [2023-04-29 09:45:10,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +25: [2023-04-29 09:45:10,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +44: [2023-04-29 09:45:10,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +44: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +46: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +30: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +44: [2023-04-29 09:45:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +53: [2023-04-29 09:45:10,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +46: [2023-04-29 09:45:10,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +53: [2023-04-29 09:45:10,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:10,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +46: [2023-04-29 09:45:10,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +46: [2023-04-29 09:45:10,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +44: [2023-04-29 09:45:10,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:10,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +30: [2023-04-29 09:45:10,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +15: [2023-04-29 09:45:10,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:10,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +46: [2023-04-29 09:45:10,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +46: [2023-04-29 09:45:10,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +46: [2023-04-29 09:45:10,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +35: [2023-04-29 09:45:10,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +35: [2023-04-29 09:45:10,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +30: [2023-04-29 09:45:10,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 4: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +35: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +11: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +35: [2023-04-29 09:45:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +35: [2023-04-29 09:45:10,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +35: [2023-04-29 09:45:10,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +35: [2023-04-29 09:45:10,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +35: [2023-04-29 09:45:10,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +15: [2023-04-29 09:45:10,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +45: [2023-04-29 09:45:10,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:10,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +35: [2023-04-29 09:45:10,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +35: [2023-04-29 09:45:10,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +35: [2023-04-29 09:45:10,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +35: [2023-04-29 09:45:10,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +11: [2023-04-29 09:45:10,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +35: [2023-04-29 09:45:10,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +63: [2023-04-29 09:45:10,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +30: [2023-04-29 09:45:10,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +35: [2023-04-29 09:45:10,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +35: [2023-04-29 09:45:10,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +45: [2023-04-29 09:45:10,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:10,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:10,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:10,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:10,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:10,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:10,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:10,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +36: [2023-04-29 09:45:10,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:10,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +30: [2023-04-29 09:45:10,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +45: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +36: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +49: [2023-04-29 09:45:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +45: [2023-04-29 09:45:10,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +30: [2023-04-29 09:45:10,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +30: [2023-04-29 09:45:10,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 5: [2023-04-29 09:45:10,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 5: [2023-04-29 09:45:10,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 8: [2023-04-29 09:45:10,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... + 5: [2023-04-29 09:45:10,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +19: [2023-04-29 09:45:10,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +30: [2023-04-29 09:45:10,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +37: [2023-04-29 09:45:10,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 4: [2023-04-29 09:45:10,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 4: [2023-04-29 09:45:10,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +49: [2023-04-29 09:45:10,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +49: [2023-04-29 09:45:10,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +27: [2023-04-29 09:45:10,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +39: [2023-04-29 09:45:10,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +19: [2023-04-29 09:45:10,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +19: [2023-04-29 09:45:10,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +19: [2023-04-29 09:45:10,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +25: [2023-04-29 09:45:10,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:10,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +49: [2023-04-29 09:45:10,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:10,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +21: [2023-04-29 09:45:10,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 9: [2023-04-29 09:45:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +40: [2023-04-29 09:45:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +27: [2023-04-29 09:45:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +46: [2023-04-29 09:45:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:10,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +18: [2023-04-29 09:45:10,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +18: [2023-04-29 09:45:10,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +39: [2023-04-29 09:45:10,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:10,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:10,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +21: [2023-04-29 09:45:10,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +21: [2023-04-29 09:45:10,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +61: [2023-04-29 09:45:10,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +40: [2023-04-29 09:45:10,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +18: [2023-04-29 09:45:10,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 0: [2023-04-29 09:45:10,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +61: [2023-04-29 09:45:10,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +44: [2023-04-29 09:45:10,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 8: [2023-04-29 09:45:10,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:10,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +37: [2023-04-29 09:45:10,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:10,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:10,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +27: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. + 3: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +55: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +13: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +25: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +25: [2023-04-29 09:45:10,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +13: [2023-04-29 09:45:10,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:10,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:10,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:10,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +25: [2023-04-29 09:45:10,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:10,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:10,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:10,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:10,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +44: [2023-04-29 09:45:10,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +35: [2023-04-29 09:45:10,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:10,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +46: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +46: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +55: [2023-04-29 09:45:10,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +39: [2023-04-29 09:45:10,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +27: [2023-04-29 09:45:10,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +35: [2023-04-29 09:45:10,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +27: [2023-04-29 09:45:10,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +25: [2023-04-29 09:45:10,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +52: [2023-04-29 09:45:10,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +21: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +35: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:10,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:10,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +39: [2023-04-29 09:45:10,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +39: [2023-04-29 09:45:10,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +49: [2023-04-29 09:45:10,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +39: [2023-04-29 09:45:10,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +43: [2023-04-29 09:45:10,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:10,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 9: [2023-04-29 09:45:10,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +46: [2023-04-29 09:45:10,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:10,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +43: [2023-04-29 09:45:10,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +52: [2023-04-29 09:45:10,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +52: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +18: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +52: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +52: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +52: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +21: [2023-04-29 09:45:10,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +21: [2023-04-29 09:45:10,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:10,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +27: [2023-04-29 09:45:10,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +39: [2023-04-29 09:45:10,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:10,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +21: [2023-04-29 09:45:10,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +44: [2023-04-29 09:45:10,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +46: [2023-04-29 09:45:10,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:10,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:10,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +53: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +37: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +61: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +47: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +18: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +47: [2023-04-29 09:45:10,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +43: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +47: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +61: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 0: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +43: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:10,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:10,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +18: [2023-04-29 09:45:10,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:10,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:10,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +40: [2023-04-29 09:45:10,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +47: [2023-04-29 09:45:10,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +47: [2023-04-29 09:45:10,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +44: [2023-04-29 09:45:10,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +47: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +25: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +47: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +47: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +25: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +46: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 5: [2023-04-29 09:45:10,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +38: [2023-04-29 09:45:10,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:10,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +53: [2023-04-29 09:45:10,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +53: [2023-04-29 09:45:10,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +44: [2023-04-29 09:45:10,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:10,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +34: [2023-04-29 09:45:10,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:10,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +35: [2023-04-29 09:45:10,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:10,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:10,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +34: [2023-04-29 09:45:10,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +34: [2023-04-29 09:45:10,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +38: [2023-04-29 09:45:10,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:10,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +38: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +38: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +38: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +38: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +38: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +32: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +31: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +32: [2023-04-29 09:45:10,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:10,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +39: [2023-04-29 09:45:10,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:10,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +39: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +31: [2023-04-29 09:45:10,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +39: [2023-04-29 09:45:10,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +19: [2023-04-29 09:45:10,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +32: [2023-04-29 09:45:10,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +33: [2023-04-29 09:45:10,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +33: [2023-04-29 09:45:10,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +32: [2023-04-29 09:45:10,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +32: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +31: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +31: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +31: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +31: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +31: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt... +32: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +41: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:10,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +31: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +31: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +31: [2023-04-29 09:45:10,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt... +42: [2023-04-29 09:45:10,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:10,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:10,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +41: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +60: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +60: [2023-04-29 09:45:10,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +42: [2023-04-29 09:45:10,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +41: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +19: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +42: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +35: [2023-04-29 09:45:10,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +41: [2023-04-29 09:45:10,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +41: [2023-04-29 09:45:10,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +41: [2023-04-29 09:45:10,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:10,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +42: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:10,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +41: [2023-04-29 09:45:11,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +42: [2023-04-29 09:45:11,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +41: [2023-04-29 09:45:11,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +42: [2023-04-29 09:45:11,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +62: [2023-04-29 09:45:11,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +19: [2023-04-29 09:45:11,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +19: [2023-04-29 09:45:11,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:11,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +62: [2023-04-29 09:45:11,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +62: [2023-04-29 09:45:11,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +62: [2023-04-29 09:45:11,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:11,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:11,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +53: [2023-04-29 09:45:11,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +62: [2023-04-29 09:45:11,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +53: [2023-04-29 09:45:11,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +62: [2023-04-29 09:45:11,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt... +62: [2023-04-29 09:45:11,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt... +43: [2023-04-29 09:45:11,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +54: [2023-04-29 09:45:11,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +54: [2023-04-29 09:45:11,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +54: [2023-04-29 09:45:11,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +19: [2023-04-29 09:45:11,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +60: [2023-04-29 09:45:11,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +60: [2023-04-29 09:45:11,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +60: [2023-04-29 09:45:11,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:11,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +55: [2023-04-29 09:45:11,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +56: [2023-04-29 09:45:11,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +56: [2023-04-29 09:45:11,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +56: [2023-04-29 09:45:11,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +56: [2023-04-29 09:45:11,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +60: [2023-04-29 09:45:11,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +43: [2023-04-29 09:45:11,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +19: [2023-04-29 09:45:11,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:11,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:11,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +52: [2023-04-29 09:45:11,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:11,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:11,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +19: [2023-04-29 09:45:11,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +43: [2023-04-29 09:45:11,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +19: [2023-04-29 09:45:11,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +43: [2023-04-29 09:45:11,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +47: [2023-04-29 09:45:11,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +47: [2023-04-29 09:45:11,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +54: [2023-04-29 09:45:11,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +47: [2023-04-29 09:45:11,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +32: [2023-04-29 09:45:11,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:11,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +54: [2023-04-29 09:45:11,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +56: [2023-04-29 09:45:11,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +56: [2023-04-29 09:45:11,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:11,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:11,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:11,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:11,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:11,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:11,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:11,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:11,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +51: [2023-04-29 09:45:11,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +51: [2023-04-29 09:45:11,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +51: [2023-04-29 09:45:11,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +38: [2023-04-29 09:45:11,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +31: [2023-04-29 09:45:11,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +51: [2023-04-29 09:45:11,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +56: [2023-04-29 09:45:11,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +56: [2023-04-29 09:45:11,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:11,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +38: [2023-04-29 09:45:11,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:11,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +31: [2023-04-29 09:45:11,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +47: [2023-04-29 09:45:11,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:11,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +52: [2023-04-29 09:45:11,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:11,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +39: [2023-04-29 09:45:11,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +39: [2023-04-29 09:45:11,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +32: [2023-04-29 09:45:11,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:11,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +38: [2023-04-29 09:45:11,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +31: [2023-04-29 09:45:11,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_00-model_states.pt. +52: [2023-04-29 09:45:11,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +41: [2023-04-29 09:45:11,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +39: [2023-04-29 09:45:11,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +28: [2023-04-29 09:45:11,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +28: [2023-04-29 09:45:11,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +28: [2023-04-29 09:45:11,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +42: [2023-04-29 09:45:11,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +16: [2023-04-29 09:45:11,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +16: [2023-04-29 09:45:11,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +16: [2023-04-29 09:45:11,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +16: [2023-04-29 09:45:11,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +42: [2023-04-29 09:45:11,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +32: [2023-04-29 09:45:11,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:11,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +47: [2023-04-29 09:45:11,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +47: [2023-04-29 09:45:11,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:11,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +57: [2023-04-29 09:45:11,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +57: [2023-04-29 09:45:11,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +57: [2023-04-29 09:45:11,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +42: [2023-04-29 09:45:11,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:11,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:11,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +62: [2023-04-29 09:45:11,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +38: [2023-04-29 09:45:11,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +53: [2023-04-29 09:45:11,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:11,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +53: [2023-04-29 09:45:11,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +51: [2023-04-29 09:45:11,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +59: [2023-04-29 09:45:11,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +31: [2023-04-29 09:45:11,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +53: [2023-04-29 09:45:11,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:11,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_00-model_states.pt. +51: [2023-04-29 09:45:11,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +31: [2023-04-29 09:45:11,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:11,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +53: [2023-04-29 09:45:11,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +53: [2023-04-29 09:45:11,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +59: [2023-04-29 09:45:11,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +38: [2023-04-29 09:45:11,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +31: [2023-04-29 09:45:11,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +41: [2023-04-29 09:45:11,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +31: [2023-04-29 09:45:11,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +53: [2023-04-29 09:45:11,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +24: [2023-04-29 09:45:11,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:11,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +41: [2023-04-29 09:45:11,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:11,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:11,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +36: [2023-04-29 09:45:11,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:11,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +16: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +36: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +16: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +36: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +24: [2023-04-29 09:45:11,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +42: [2023-04-29 09:45:11,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +42: [2023-04-29 09:45:11,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +41: [2023-04-29 09:45:11,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +24: [2023-04-29 09:45:11,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +45: [2023-04-29 09:45:11,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +33: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +33: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +24: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +24: [2023-04-29 09:45:11,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +36: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +36: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +36: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +50: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +33: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +63: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +63: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +63: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +57: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:11,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +36: [2023-04-29 09:45:11,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +57: [2023-04-29 09:45:11,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:11,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +57: [2023-04-29 09:45:11,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:11,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:11,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +33: [2023-04-29 09:45:11,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:11,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:11,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:11,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:11,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:11,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:11,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:11,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +42: [2023-04-29 09:45:11,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +50: [2023-04-29 09:45:11,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +33: [2023-04-29 09:45:11,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +33: [2023-04-29 09:45:11,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +61: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +61: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +33: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +59: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +62: [2023-04-29 09:45:11,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +49: [2023-04-29 09:45:11,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +49: [2023-04-29 09:45:11,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +59: [2023-04-29 09:45:11,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +24: [2023-04-29 09:45:11,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +63: [2023-04-29 09:45:11,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:11,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:11,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +29: [2023-04-29 09:45:11,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:11,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:11,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +24: [2023-04-29 09:45:11,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +50: [2023-04-29 09:45:11,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +51: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +51: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +58: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +58: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +58: [2023-04-29 09:45:11,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +44: [2023-04-29 09:45:11,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +44: [2023-04-29 09:45:11,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +44: [2023-04-29 09:45:11,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +29: [2023-04-29 09:45:11,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +61: [2023-04-29 09:45:11,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +29: [2023-04-29 09:45:11,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +61: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +46: [2023-04-29 09:45:11,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +46: [2023-04-29 09:45:11,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +46: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +16: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +46: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +16: [2023-04-29 09:45:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +51: [2023-04-29 09:45:11,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +35: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +58: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +51: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +49: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +35: [2023-04-29 09:45:11,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +51: [2023-04-29 09:45:11,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +58: [2023-04-29 09:45:11,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +58: [2023-04-29 09:45:11,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +58: [2023-04-29 09:45:11,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +49: [2023-04-29 09:45:11,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:11,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +16: [2023-04-29 09:45:11,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +49: [2023-04-29 09:45:11,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:11,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:11,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +51: [2023-04-29 09:45:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +16: [2023-04-29 09:45:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +40: [2023-04-29 09:45:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +40: [2023-04-29 09:45:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +40: [2023-04-29 09:45:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +51: [2023-04-29 09:45:11,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +44: [2023-04-29 09:45:11,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:11,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +36: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +16: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +44: [2023-04-29 09:45:11,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +24: [2023-04-29 09:45:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +46: [2023-04-29 09:45:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +46: [2023-04-29 09:45:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +46: [2023-04-29 09:45:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +36: [2023-04-29 09:45:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:11,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +16: [2023-04-29 09:45:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +58: [2023-04-29 09:45:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +33: [2023-04-29 09:45:11,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +51: [2023-04-29 09:45:11,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +55: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +55: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +55: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +16: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +35: [2023-04-29 09:45:11,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +58: [2023-04-29 09:45:11,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +16: [2023-04-29 09:45:11,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +52: [2023-04-29 09:45:11,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +52: [2023-04-29 09:45:11,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +52: [2023-04-29 09:45:11,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +52: [2023-04-29 09:45:11,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +16: [2023-04-29 09:45:11,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +40: [2023-04-29 09:45:11,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +40: [2023-04-29 09:45:11,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +33: [2023-04-29 09:45:11,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +40: [2023-04-29 09:45:11,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:11,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:11,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +24: [2023-04-29 09:45:11,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +36: [2023-04-29 09:45:11,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +20: [2023-04-29 09:45:11,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +14: [2023-04-29 09:45:11,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +36: [2023-04-29 09:45:11,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +33: [2023-04-29 09:45:11,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +24: [2023-04-29 09:45:11,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +33: [2023-04-29 09:45:11,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +52: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:11,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +23: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +23: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +48: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +48: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +23: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +20: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +52: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +29: [2023-04-29 09:45:11,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +23: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +20: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +20: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +45: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +20: [2023-04-29 09:45:11,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +48: [2023-04-29 09:45:11,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +48: [2023-04-29 09:45:11,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +51: [2023-04-29 09:45:11,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +26: [2023-04-29 09:45:11,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +26: [2023-04-29 09:45:11,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +26: [2023-04-29 09:45:11,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +26: [2023-04-29 09:45:11,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +29: [2023-04-29 09:45:11,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +29: [2023-04-29 09:45:11,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 4: [2023-04-29 09:45:11,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 4: [2023-04-29 09:45:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 4: [2023-04-29 09:45:11,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +47: [2023-04-29 09:45:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +47: [2023-04-29 09:45:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +47: [2023-04-29 09:45:11,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +29: [2023-04-29 09:45:11,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +55: [2023-04-29 09:45:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +55: [2023-04-29 09:45:11,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +47: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +58: [2023-04-29 09:45:11,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +58: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +58: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +16: [2023-04-29 09:45:11,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +14: [2023-04-29 09:45:11,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:11,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:11,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +49: [2023-04-29 09:45:11,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +45: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +45: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +59: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +45: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +45: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +50: [2023-04-29 09:45:11,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +24: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +24: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +59: [2023-04-29 09:45:11,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +59: [2023-04-29 09:45:11,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +50: [2023-04-29 09:45:11,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +50: [2023-04-29 09:45:11,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +50: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +49: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +59: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +49: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +49: [2023-04-29 09:45:11,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +51: [2023-04-29 09:45:11,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:11,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +37: [2023-04-29 09:45:11,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +49: [2023-04-29 09:45:11,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +14: [2023-04-29 09:45:11,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +37: [2023-04-29 09:45:11,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +37: [2023-04-29 09:45:11,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +49: [2023-04-29 09:45:11,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +37: [2023-04-29 09:45:11,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +58: [2023-04-29 09:45:11,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +23: [2023-04-29 09:45:11,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +20: [2023-04-29 09:45:11,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +43: [2023-04-29 09:45:11,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +16: [2023-04-29 09:45:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +43: [2023-04-29 09:45:11,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +43: [2023-04-29 09:45:11,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +43: [2023-04-29 09:45:11,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +29: [2023-04-29 09:45:11,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 7: [2023-04-29 09:45:11,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +48: [2023-04-29 09:45:11,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +51: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +58: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +48: [2023-04-29 09:45:11,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +11: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +11: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +11: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +11: [2023-04-29 09:45:11,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +48: [2023-04-29 09:45:11,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +26: [2023-04-29 09:45:11,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +20: [2023-04-29 09:45:11,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +23: [2023-04-29 09:45:11,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 2: [2023-04-29 09:45:11,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 2: [2023-04-29 09:45:11,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 2: [2023-04-29 09:45:11,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +58: [2023-04-29 09:45:11,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +26: [2023-04-29 09:45:11,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +29: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +47: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +10: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +58: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +10: [2023-04-29 09:45:11,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +20: [2023-04-29 09:45:11,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:11,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +47: [2023-04-29 09:45:11,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +58: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +29: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +19: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +19: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +47: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +19: [2023-04-29 09:45:11,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +17: [2023-04-29 09:45:11,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +17: [2023-04-29 09:45:11,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +17: [2023-04-29 09:45:11,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +17: [2023-04-29 09:45:11,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +58: [2023-04-29 09:45:11,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +14: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +47: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +14: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +14: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:11,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:11,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:11,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:11,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +16: [2023-04-29 09:45:11,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +28: [2023-04-29 09:45:11,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +14: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +62: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +62: [2023-04-29 09:45:11,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:11,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_34-model_01-model_states.pt. +45: [2023-04-29 09:45:11,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +16: [2023-04-29 09:45:11,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +57: [2023-04-29 09:45:11,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +14: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +58: [2023-04-29 09:45:11,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:11,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +43: [2023-04-29 09:45:11,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +28: [2023-04-29 09:45:11,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +43: [2023-04-29 09:45:11,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +40: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +28: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +57: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +57: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +40: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +57: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +20: [2023-04-29 09:45:11,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +57: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +28: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +11: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +57: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +28: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +57: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +57: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +43: [2023-04-29 09:45:11,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:11,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +40: [2023-04-29 09:45:11,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +58: [2023-04-29 09:45:11,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +34: [2023-04-29 09:45:11,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +34: [2023-04-29 09:45:11,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +34: [2023-04-29 09:45:11,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 2: [2023-04-29 09:45:11,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:11,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... + 2: [2023-04-29 09:45:11,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:11,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +11: [2023-04-29 09:45:11,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:11,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +27: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +27: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +13: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +13: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +27: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +13: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +13: [2023-04-29 09:45:11,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +10: [2023-04-29 09:45:11,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:11,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +19: [2023-04-29 09:45:11,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +19: [2023-04-29 09:45:11,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +19: [2023-04-29 09:45:11,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +19: [2023-04-29 09:45:11,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:11,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +62: [2023-04-29 09:45:11,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +10: [2023-04-29 09:45:11,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +10: [2023-04-29 09:45:11,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:11,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +10: [2023-04-29 09:45:11,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +10: [2023-04-29 09:45:11,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +42: [2023-04-29 09:45:11,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +42: [2023-04-29 09:45:11,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +49: [2023-04-29 09:45:11,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +14: [2023-04-29 09:45:11,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +44: [2023-04-29 09:45:11,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +10: [2023-04-29 09:45:11,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +54: [2023-04-29 09:45:11,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +42: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +44: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +20: [2023-04-29 09:45:11,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +10: [2023-04-29 09:45:11,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:11,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +10: [2023-04-29 09:45:11,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +62: [2023-04-29 09:45:11,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +48: [2023-04-29 09:45:11,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +50: [2023-04-29 09:45:11,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:11,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:11,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +34: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +54: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +54: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +34: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +41: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +41: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +48: [2023-04-29 09:45:11,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +34: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +44: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +44: [2023-04-29 09:45:11,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:11,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +59: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +13: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +27: [2023-04-29 09:45:11,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +60: [2023-04-29 09:45:11,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +41: [2023-04-29 09:45:11,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +30: [2023-04-29 09:45:11,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +30: [2023-04-29 09:45:11,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +54: [2023-04-29 09:45:11,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +13: [2023-04-29 09:45:11,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +54: [2023-04-29 09:45:11,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +49: [2023-04-29 09:45:11,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +13: [2023-04-29 09:45:11,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +54: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +13: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +27: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +54: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +50: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +20: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +30: [2023-04-29 09:45:11,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +27: [2023-04-29 09:45:11,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +60: [2023-04-29 09:45:11,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +61: [2023-04-29 09:45:11,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:11,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +60: [2023-04-29 09:45:11,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +60: [2023-04-29 09:45:11,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +20: [2023-04-29 09:45:11,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +60: [2023-04-29 09:45:11,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +27: [2023-04-29 09:45:11,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +60: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +56: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +61: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +61: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +60: [2023-04-29 09:45:11,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +56: [2023-04-29 09:45:11,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +22: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +46: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +45: [2023-04-29 09:45:11,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +48: [2023-04-29 09:45:11,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +46: [2023-04-29 09:45:11,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +46: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +46: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +20: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +48: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +56: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +56: [2023-04-29 09:45:11,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +61: [2023-04-29 09:45:11,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +61: [2023-04-29 09:45:11,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +22: [2023-04-29 09:45:11,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +61: [2023-04-29 09:45:11,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +12: [2023-04-29 09:45:11,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +12: [2023-04-29 09:45:11,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +46: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +14: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +46: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +46: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +12: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +46: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +14: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +12: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +46: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +45: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +46: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +20: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +42: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +42: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +46: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +46: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +42: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +42: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +48: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +63: [2023-04-29 09:45:11,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:11,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +50: [2023-04-29 09:45:11,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +14: [2023-04-29 09:45:11,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +12: [2023-04-29 09:45:11,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +12: [2023-04-29 09:45:11,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +28: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +38: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +38: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +38: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +38: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +59: [2023-04-29 09:45:11,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +12: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +12: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +63: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +63: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +59: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +63: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +17: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +17: [2023-04-29 09:45:11,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +40: [2023-04-29 09:45:11,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +14: [2023-04-29 09:45:11,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +45: [2023-04-29 09:45:11,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +15: [2023-04-29 09:45:11,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +15: [2023-04-29 09:45:11,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +63: [2023-04-29 09:45:11,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +59: [2023-04-29 09:45:11,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +63: [2023-04-29 09:45:11,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +14: [2023-04-29 09:45:11,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +15: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +14: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +30: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +12: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +57: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +41: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +15: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +12: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +41: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +45: [2023-04-29 09:45:11,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +49: [2023-04-29 09:45:11,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +63: [2023-04-29 09:45:11,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:11,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +63: [2023-04-29 09:45:11,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +35: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:11,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +63: [2023-04-29 09:45:11,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +41: [2023-04-29 09:45:11,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +59: [2023-04-29 09:45:11,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +30: [2023-04-29 09:45:11,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +50: [2023-04-29 09:45:11,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +45: [2023-04-29 09:45:11,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +63: [2023-04-29 09:45:11,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +39: [2023-04-29 09:45:11,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +30: [2023-04-29 09:45:11,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +45: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +30: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +49: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +35: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +57: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +49: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +35: [2023-04-29 09:45:11,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +22: [2023-04-29 09:45:11,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +22: [2023-04-29 09:45:11,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +22: [2023-04-29 09:45:11,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +35: [2023-04-29 09:45:11,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +22: [2023-04-29 09:45:11,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +20: [2023-04-29 09:45:11,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +20: [2023-04-29 09:45:11,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +57: [2023-04-29 09:45:11,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:11,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +39: [2023-04-29 09:45:11,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +40: [2023-04-29 09:45:11,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +28: [2023-04-29 09:45:11,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +14: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +37: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +35: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +35: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +35: [2023-04-29 09:45:11,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +35: [2023-04-29 09:45:11,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +39: [2023-04-29 09:45:11,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +20: [2023-04-29 09:45:11,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +39: [2023-04-29 09:45:11,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +28: [2023-04-29 09:45:11,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +39: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +39: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +32: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +32: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +32: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +37: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +37: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +32: [2023-04-29 09:45:11,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +37: [2023-04-29 09:45:11,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +37: [2023-04-29 09:45:11,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +17: [2023-04-29 09:45:11,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +37: [2023-04-29 09:45:11,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +37: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +28: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +17: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +37: [2023-04-29 09:45:11,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +37: [2023-04-29 09:45:11,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +39: [2023-04-29 09:45:11,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +39: [2023-04-29 09:45:11,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +40: [2023-04-29 09:45:11,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:11,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +20: [2023-04-29 09:45:11,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +14: [2023-04-29 09:45:11,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +14: [2023-04-29 09:45:11,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +38: [2023-04-29 09:45:11,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +14: [2023-04-29 09:45:11,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 8: [2023-04-29 09:45:11,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 8: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +17: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +12: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +17: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +15: [2023-04-29 09:45:11,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +20: [2023-04-29 09:45:11,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +52: [2023-04-29 09:45:11,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +10: [2023-04-29 09:45:11,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +44: [2023-04-29 09:45:11,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:11,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +57: [2023-04-29 09:45:11,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +20: [2023-04-29 09:45:11,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +12: [2023-04-29 09:45:11,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +44: [2023-04-29 09:45:11,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +52: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +57: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +52: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +57: [2023-04-29 09:45:11,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +40: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +52: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +10: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +55: [2023-04-29 09:45:11,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +53: [2023-04-29 09:45:11,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +55: [2023-04-29 09:45:11,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +44: [2023-04-29 09:45:11,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:11,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:11,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +53: [2023-04-29 09:45:11,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +60: [2023-04-29 09:45:11,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +22: [2023-04-29 09:45:11,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +22: [2023-04-29 09:45:11,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +32: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +10: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +47: [2023-04-29 09:45:11,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +56: [2023-04-29 09:45:11,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +56: [2023-04-29 09:45:11,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +28: [2023-04-29 09:45:11,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +32: [2023-04-29 09:45:11,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +54: [2023-04-29 09:45:11,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +48: [2023-04-29 09:45:11,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +54: [2023-04-29 09:45:11,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +54: [2023-04-29 09:45:11,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +32: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +26: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +47: [2023-04-29 09:45:11,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +56: [2023-04-29 09:45:11,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +10: [2023-04-29 09:45:11,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +47: [2023-04-29 09:45:11,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +44: [2023-04-29 09:45:11,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +56: [2023-04-29 09:45:11,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +61: [2023-04-29 09:45:11,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +29: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +29: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +29: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +60: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +61: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:11,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +43: [2023-04-29 09:45:11,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +43: [2023-04-29 09:45:11,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +46: [2023-04-29 09:45:11,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +10: [2023-04-29 09:45:11,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:11,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +43: [2023-04-29 09:45:11,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +10: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +43: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +43: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +43: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +43: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +46: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +40: [2023-04-29 09:45:11,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +44: [2023-04-29 09:45:11,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +61: [2023-04-29 09:45:11,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +22: [2023-04-29 09:45:11,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +14: [2023-04-29 09:45:11,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +46: [2023-04-29 09:45:11,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +12: [2023-04-29 09:45:11,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +46: [2023-04-29 09:45:11,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +26: [2023-04-29 09:45:11,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +43: [2023-04-29 09:45:11,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +26: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +17: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +17: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +17: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +17: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +43: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:11,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +22: [2023-04-29 09:45:11,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +26: [2023-04-29 09:45:11,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +43: [2023-04-29 09:45:11,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +43: [2023-04-29 09:45:11,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +43: [2023-04-29 09:45:11,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +48: [2023-04-29 09:45:11,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +48: [2023-04-29 09:45:11,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +38: [2023-04-29 09:45:11,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +38: [2023-04-29 09:45:11,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +38: [2023-04-29 09:45:11,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +22: [2023-04-29 09:45:11,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +44: [2023-04-29 09:45:11,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +63: [2023-04-29 09:45:11,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +63: [2023-04-29 09:45:11,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +10: [2023-04-29 09:45:11,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +35: [2023-04-29 09:45:11,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +63: [2023-04-29 09:45:11,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +60: [2023-04-29 09:45:11,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +56: [2023-04-29 09:45:11,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +60: [2023-04-29 09:45:11,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +56: [2023-04-29 09:45:11,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +38: [2023-04-29 09:45:11,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +23: [2023-04-29 09:45:11,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +12: [2023-04-29 09:45:11,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +61: [2023-04-29 09:45:11,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +34: [2023-04-29 09:45:11,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +23: [2023-04-29 09:45:11,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +54: [2023-04-29 09:45:11,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +23: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +23: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +12: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +32: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +23: [2023-04-29 09:45:11,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +34: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +61: [2023-04-29 09:45:11,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +46: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +46: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +29: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +29: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +60: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +56: [2023-04-29 09:45:11,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +54: [2023-04-29 09:45:11,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +54: [2023-04-29 09:45:11,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +37: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +34: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +61: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +34: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +35: [2023-04-29 09:45:11,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +34: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +34: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +34: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +56: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +10: [2023-04-29 09:45:11,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +34: [2023-04-29 09:45:11,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +34: [2023-04-29 09:45:11,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +46: [2023-04-29 09:45:11,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +46: [2023-04-29 09:45:11,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +22: [2023-04-29 09:45:11,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +38: [2023-04-29 09:45:11,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +38: [2023-04-29 09:45:11,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:11,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +39: [2023-04-29 09:45:11,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 2: [2023-04-29 09:45:11,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 2: [2023-04-29 09:45:11,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 2: [2023-04-29 09:45:11,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +37: [2023-04-29 09:45:11,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +41: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +35: [2023-04-29 09:45:11,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:11,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +18: [2023-04-29 09:45:11,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +63: [2023-04-29 09:45:11,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +38: [2023-04-29 09:45:11,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +38: [2023-04-29 09:45:11,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +38: [2023-04-29 09:45:11,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +18: [2023-04-29 09:45:11,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +18: [2023-04-29 09:45:11,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +41: [2023-04-29 09:45:11,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +41: [2023-04-29 09:45:11,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +53: [2023-04-29 09:45:11,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +53: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +53: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +53: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +63: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +41: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +41: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +15: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +15: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +15: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +15: [2023-04-29 09:45:11,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +12: [2023-04-29 09:45:11,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +41: [2023-04-29 09:45:11,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +41: [2023-04-29 09:45:11,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +52: [2023-04-29 09:45:11,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +21: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +12: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +32: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +21: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +21: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +41: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +41: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +15: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +15: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +15: [2023-04-29 09:45:11,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +15: [2023-04-29 09:45:11,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +52: [2023-04-29 09:45:11,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +52: [2023-04-29 09:45:11,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +12: [2023-04-29 09:45:11,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +38: [2023-04-29 09:45:11,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +38: [2023-04-29 09:45:11,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +55: [2023-04-29 09:45:11,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +53: [2023-04-29 09:45:11,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +12: [2023-04-29 09:45:11,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +47: [2023-04-29 09:45:11,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +12: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +36: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +36: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +36: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +39: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +15: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +36: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +15: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +15: [2023-04-29 09:45:11,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +15: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +15: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +12: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +15: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +37: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +12: [2023-04-29 09:45:11,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 9: [2023-04-29 09:45:11,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +26: [2023-04-29 09:45:11,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:11,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 5: [2023-04-29 09:45:11,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 5: [2023-04-29 09:45:11,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +33: [2023-04-29 09:45:11,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +33: [2023-04-29 09:45:11,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +12: [2023-04-29 09:45:11,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +22: [2023-04-29 09:45:11,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 5: [2023-04-29 09:45:11,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 5: [2023-04-29 09:45:11,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +53: [2023-04-29 09:45:11,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +47: [2023-04-29 09:45:11,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +22: [2023-04-29 09:45:11,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +33: [2023-04-29 09:45:11,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +39: [2023-04-29 09:45:11,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +39: [2023-04-29 09:45:11,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:11,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +39: [2023-04-29 09:45:11,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +22: [2023-04-29 09:45:11,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +53: [2023-04-29 09:45:11,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +21: [2023-04-29 09:45:11,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +37: [2023-04-29 09:45:11,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +26: [2023-04-29 09:45:11,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +26: [2023-04-29 09:45:11,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +39: [2023-04-29 09:45:11,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +21: [2023-04-29 09:45:11,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +22: [2023-04-29 09:45:11,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +43: [2023-04-29 09:45:11,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +25: [2023-04-29 09:45:11,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +25: [2023-04-29 09:45:11,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +39: [2023-04-29 09:45:11,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +18: [2023-04-29 09:45:11,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +37: [2023-04-29 09:45:11,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +32: [2023-04-29 09:45:11,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +18: [2023-04-29 09:45:11,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +39: [2023-04-29 09:45:11,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +25: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +21: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +25: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +21: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +21: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +53: [2023-04-29 09:45:11,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +53: [2023-04-29 09:45:11,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +43: [2023-04-29 09:45:11,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +43: [2023-04-29 09:45:11,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +22: [2023-04-29 09:45:11,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +39: [2023-04-29 09:45:11,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +21: [2023-04-29 09:45:11,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +21: [2023-04-29 09:45:11,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +23: [2023-04-29 09:45:11,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:11,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +21: [2023-04-29 09:45:11,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +21: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +39: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +52: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +52: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +55: [2023-04-29 09:45:11,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +32: [2023-04-29 09:45:11,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +32: [2023-04-29 09:45:11,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +55: [2023-04-29 09:45:11,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +26: [2023-04-29 09:45:11,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +47: [2023-04-29 09:45:11,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +39: [2023-04-29 09:45:11,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +36: [2023-04-29 09:45:11,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +38: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +23: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +23: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +62: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +38: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +53: [2023-04-29 09:45:11,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +38: [2023-04-29 09:45:11,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +31: [2023-04-29 09:45:11,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +31: [2023-04-29 09:45:11,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +53: [2023-04-29 09:45:11,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:11,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +62: [2023-04-29 09:45:11,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +47: [2023-04-29 09:45:11,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +29: [2023-04-29 09:45:11,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +62: [2023-04-29 09:45:11,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +32: [2023-04-29 09:45:11,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:11,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt... +34: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +42: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +29: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +62: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +23: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +31: [2023-04-29 09:45:11,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +34: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +31: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +62: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt... +42: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +42: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +43: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +42: [2023-04-29 09:45:11,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +29: [2023-04-29 09:45:11,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +42: [2023-04-29 09:45:11,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +25: [2023-04-29 09:45:11,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +17: [2023-04-29 09:45:11,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +42: [2023-04-29 09:45:11,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +42: [2023-04-29 09:45:11,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +43: [2023-04-29 09:45:11,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +25: [2023-04-29 09:45:11,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +25: [2023-04-29 09:45:11,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +26: [2023-04-29 09:45:11,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +29: [2023-04-29 09:45:11,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +26: [2023-04-29 09:45:11,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +17: [2023-04-29 09:45:11,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +29: [2023-04-29 09:45:11,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +29: [2023-04-29 09:45:11,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +32: [2023-04-29 09:45:11,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +43: [2023-04-29 09:45:11,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +17: [2023-04-29 09:45:11,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +43: [2023-04-29 09:45:11,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +23: [2023-04-29 09:45:11,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +32: [2023-04-29 09:45:11,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +18: [2023-04-29 09:45:11,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +18: [2023-04-29 09:45:11,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +38: [2023-04-29 09:45:11,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +18: [2023-04-29 09:45:11,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +18: [2023-04-29 09:45:11,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +41: [2023-04-29 09:45:11,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +17: [2023-04-29 09:45:11,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +38: [2023-04-29 09:45:11,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +18: [2023-04-29 09:45:11,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +41: [2023-04-29 09:45:11,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +18: [2023-04-29 09:45:11,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +38: [2023-04-29 09:45:11,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +23: [2023-04-29 09:45:11,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:11,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +15: [2023-04-29 09:45:11,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +18: [2023-04-29 09:45:11,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +34: [2023-04-29 09:45:11,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +31: [2023-04-29 09:45:11,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:11,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +31: [2023-04-29 09:45:11,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +31: [2023-04-29 09:45:11,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +32: [2023-04-29 09:45:11,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:11,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +31: [2023-04-29 09:45:11,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +15: [2023-04-29 09:45:11,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +34: [2023-04-29 09:45:11,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +15: [2023-04-29 09:45:11,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +34: [2023-04-29 09:45:11,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:11,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +32: [2023-04-29 09:45:11,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +11: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +34: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +23: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +21: [2023-04-29 09:45:11,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 0: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +25: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:11,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:11,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +11: [2023-04-29 09:45:11,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 0: [2023-04-29 09:45:11,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:11,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:11,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:11,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:11,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +25: [2023-04-29 09:45:11,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 8: [2023-04-29 09:45:11,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 8: [2023-04-29 09:45:11,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +32: [2023-04-29 09:45:11,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 8: [2023-04-29 09:45:11,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +33: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +33: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +33: [2023-04-29 09:45:11,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +33: [2023-04-29 09:45:11,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +25: [2023-04-29 09:45:11,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +25: [2023-04-29 09:45:11,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +41: [2023-04-29 09:45:11,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +53: [2023-04-29 09:45:11,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +53: [2023-04-29 09:45:11,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:11,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +25: [2023-04-29 09:45:11,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +33: [2023-04-29 09:45:11,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +62: [2023-04-29 09:45:11,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +21: [2023-04-29 09:45:11,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +21: [2023-04-29 09:45:11,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +53: [2023-04-29 09:45:11,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +18: [2023-04-29 09:45:11,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +16: [2023-04-29 09:45:11,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +16: [2023-04-29 09:45:11,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +15: [2023-04-29 09:45:11,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +53: [2023-04-29 09:45:11,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +15: [2023-04-29 09:45:11,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:11,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +21: [2023-04-29 09:45:11,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +18: [2023-04-29 09:45:11,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 7: [2023-04-29 09:45:11,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 7: [2023-04-29 09:45:11,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_13-model_01-model_states.pt. +21: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +53: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +18: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 3: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +53: [2023-04-29 09:45:11,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +18: [2023-04-29 09:45:11,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +42: [2023-04-29 09:45:11,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +62: [2023-04-29 09:45:11,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +15: [2023-04-29 09:45:11,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +15: [2023-04-29 09:45:11,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:11,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +19: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +42: [2023-04-29 09:45:11,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +21: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +53: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +45: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +30: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:11,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +30: [2023-04-29 09:45:11,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +19: [2023-04-29 09:45:11,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +18: [2023-04-29 09:45:11,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +42: [2023-04-29 09:45:11,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +45: [2023-04-29 09:45:11,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +33: [2023-04-29 09:45:11,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +33: [2023-04-29 09:45:11,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +30: [2023-04-29 09:45:11,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +45: [2023-04-29 09:45:11,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +30: [2023-04-29 09:45:11,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +30: [2023-04-29 09:45:11,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +30: [2023-04-29 09:45:11,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +30: [2023-04-29 09:45:11,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +42: [2023-04-29 09:45:11,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +30: [2023-04-29 09:45:11,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +33: [2023-04-29 09:45:11,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +62: [2023-04-29 09:45:11,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +30: [2023-04-29 09:45:11,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +30: [2023-04-29 09:45:11,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +19: [2023-04-29 09:45:11,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +62: [2023-04-29 09:45:11,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +25: [2023-04-29 09:45:11,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +19: [2023-04-29 09:45:11,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +19: [2023-04-29 09:45:11,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +62: [2023-04-29 09:45:11,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_00-model_states.pt. +19: [2023-04-29 09:45:11,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +19: [2023-04-29 09:45:11,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +27: [2023-04-29 09:45:11,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +21: [2023-04-29 09:45:11,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:11,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +27: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +27: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +16: [2023-04-29 09:45:11,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +18: [2023-04-29 09:45:11,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +13: [2023-04-29 09:45:11,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +27: [2023-04-29 09:45:11,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +13: [2023-04-29 09:45:11,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +21: [2023-04-29 09:45:11,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:11,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +62: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +31: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +31: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +58: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +13: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +31: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +31: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +27: [2023-04-29 09:45:11,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +27: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +27: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +13: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +27: [2023-04-29 09:45:11,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +13: [2023-04-29 09:45:11,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +27: [2023-04-29 09:45:11,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 3: [2023-04-29 09:45:11,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +33: [2023-04-29 09:45:11,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +27: [2023-04-29 09:45:11,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +21: [2023-04-29 09:45:11,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:11,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +51: [2023-04-29 09:45:11,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:11,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +42: [2023-04-29 09:45:11,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +31: [2023-04-29 09:45:11,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +31: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +58: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +58: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +31: [2023-04-29 09:45:11,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +31: [2023-04-29 09:45:11,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +31: [2023-04-29 09:45:11,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +25: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +18: [2023-04-29 09:45:11,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +31: [2023-04-29 09:45:11,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +18: [2023-04-29 09:45:11,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +31: [2023-04-29 09:45:11,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +42: [2023-04-29 09:45:11,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +42: [2023-04-29 09:45:11,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +51: [2023-04-29 09:45:11,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +31: [2023-04-29 09:45:11,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +31: [2023-04-29 09:45:11,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +18: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +31: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +25: [2023-04-29 09:45:11,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 5: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 5: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +51: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +25: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +51: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +50: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +50: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +51: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +51: [2023-04-29 09:45:11,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 5: [2023-04-29 09:45:11,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +51: [2023-04-29 09:45:11,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 5: [2023-04-29 09:45:11,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +36: [2023-04-29 09:45:11,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +36: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +59: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +36: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +50: [2023-04-29 09:45:11,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +59: [2023-04-29 09:45:11,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +11: [2023-04-29 09:45:11,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +59: [2023-04-29 09:45:11,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +25: [2023-04-29 09:45:11,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +62: [2023-04-29 09:45:11,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +31: [2023-04-29 09:45:11,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +31: [2023-04-29 09:45:11,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 8: [2023-04-29 09:45:11,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +11: [2023-04-29 09:45:11,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +36: [2023-04-29 09:45:11,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +36: [2023-04-29 09:45:11,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +36: [2023-04-29 09:45:11,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +51: [2023-04-29 09:45:11,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:11,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +36: [2023-04-29 09:45:11,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +49: [2023-04-29 09:45:11,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +49: [2023-04-29 09:45:11,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +49: [2023-04-29 09:45:11,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +49: [2023-04-29 09:45:11,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +49: [2023-04-29 09:45:11,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +49: [2023-04-29 09:45:11,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +18: [2023-04-29 09:45:11,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +49: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +33: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +51: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +24: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +58: [2023-04-29 09:45:11,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +58: [2023-04-29 09:45:11,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +58: [2023-04-29 09:45:11,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:11,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +40: [2023-04-29 09:45:11,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +40: [2023-04-29 09:45:11,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +54: [2023-04-29 09:45:11,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +54: [2023-04-29 09:45:11,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +25: [2023-04-29 09:45:11,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +40: [2023-04-29 09:45:11,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +54: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +51: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +24: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +11: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt... +11: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +33: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 9: [2023-04-29 09:45:11,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +59: [2023-04-29 09:45:11,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +25: [2023-04-29 09:45:11,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +59: [2023-04-29 09:45:11,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +59: [2023-04-29 09:45:11,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:11,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 8: [2023-04-29 09:45:11,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +11: [2023-04-29 09:45:11,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +30: [2023-04-29 09:45:11,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +50: [2023-04-29 09:45:11,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +44: [2023-04-29 09:45:11,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +45: [2023-04-29 09:45:11,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:11,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:11,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:11,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +51: [2023-04-29 09:45:11,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +19: [2023-04-29 09:45:11,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +51: [2023-04-29 09:45:11,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:11,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +28: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +24: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +28: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +28: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +24: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +28: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +30: [2023-04-29 09:45:11,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:11,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +44: [2023-04-29 09:45:11,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +13: [2023-04-29 09:45:11,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +27: [2023-04-29 09:45:11,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +13: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +45: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 3: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +33: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +19: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +27: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +20: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +20: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +58: [2023-04-29 09:45:11,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:11,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +57: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +57: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +58: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +25: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +20: [2023-04-29 09:45:11,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +58: [2023-04-29 09:45:11,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +31: [2023-04-29 09:45:11,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +54: [2023-04-29 09:45:11,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +54: [2023-04-29 09:45:11,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +54: [2023-04-29 09:45:11,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +58: [2023-04-29 09:45:11,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +16: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +58: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +61: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +61: [2023-04-29 09:45:11,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +61: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +61: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +44: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +16: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +16: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +49: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +49: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +49: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +16: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +16: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +16: [2023-04-29 09:45:11,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +49: [2023-04-29 09:45:11,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +16: [2023-04-29 09:45:11,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +24: [2023-04-29 09:45:11,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +45: [2023-04-29 09:45:11,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +30: [2023-04-29 09:45:11,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +44: [2023-04-29 09:45:11,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:11,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:11,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +24: [2023-04-29 09:45:11,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +59: [2023-04-29 09:45:11,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +35: [2023-04-29 09:45:11,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +16: [2023-04-29 09:45:11,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +28: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +36: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +16: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +58: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:11,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +27: [2023-04-29 09:45:11,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:11,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +35: [2023-04-29 09:45:11,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +31: [2023-04-29 09:45:11,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +44: [2023-04-29 09:45:11,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +14: [2023-04-29 09:45:11,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +14: [2023-04-29 09:45:11,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:11,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:11,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:11,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +13: [2023-04-29 09:45:11,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +27: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +14: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +36: [2023-04-29 09:45:11,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +28: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +14: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +49: [2023-04-29 09:45:11,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +57: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +19: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +28: [2023-04-29 09:45:11,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +50: [2023-04-29 09:45:11,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +49: [2023-04-29 09:45:11,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +31: [2023-04-29 09:45:11,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +19: [2023-04-29 09:45:11,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +58: [2023-04-29 09:45:11,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +50: [2023-04-29 09:45:11,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +36: [2023-04-29 09:45:11,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +31: [2023-04-29 09:45:11,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +58: [2023-04-29 09:45:11,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +61: [2023-04-29 09:45:11,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +61: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 4: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +49: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +61: [2023-04-29 09:45:11,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +61: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +48: [2023-04-29 09:45:11,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:11,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +17: [2023-04-29 09:45:11,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +17: [2023-04-29 09:45:11,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +17: [2023-04-29 09:45:11,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +17: [2023-04-29 09:45:11,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +36: [2023-04-29 09:45:11,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 5: [2023-04-29 09:45:11,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +48: [2023-04-29 09:45:11,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +59: [2023-04-29 09:45:11,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +24: [2023-04-29 09:45:11,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... + 7: [2023-04-29 09:45:11,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +49: [2023-04-29 09:45:11,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt... +44: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +47: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +47: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +47: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +47: [2023-04-29 09:45:11,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_00-model_states.pt. +44: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +52: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +51: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +29: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +35: [2023-04-29 09:45:11,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +17: [2023-04-29 09:45:11,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +17: [2023-04-29 09:45:11,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +52: [2023-04-29 09:45:11,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +52: [2023-04-29 09:45:11,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +58: [2023-04-29 09:45:11,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:11,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:11,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:11,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:11,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +35: [2023-04-29 09:45:11,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +29: [2023-04-29 09:45:11,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +44: [2023-04-29 09:45:11,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +49: [2023-04-29 09:45:11,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +51: [2023-04-29 09:45:11,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +58: [2023-04-29 09:45:11,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:11,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +56: [2023-04-29 09:45:11,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +56: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +59: [2023-04-29 09:45:11,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +56: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +56: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +14: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 7: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +36: [2023-04-29 09:45:11,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +29: [2023-04-29 09:45:11,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +51: [2023-04-29 09:45:11,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +14: [2023-04-29 09:45:11,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +29: [2023-04-29 09:45:11,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +14: [2023-04-29 09:45:11,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +37: [2023-04-29 09:45:11,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +37: [2023-04-29 09:45:11,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +36: [2023-04-29 09:45:11,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +63: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +63: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +49: [2023-04-29 09:45:11,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +37: [2023-04-29 09:45:11,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +51: [2023-04-29 09:45:11,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +49: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +59: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +20: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +32: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +32: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +32: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +32: [2023-04-29 09:45:11,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +51: [2023-04-29 09:45:11,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +24: [2023-04-29 09:45:11,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +59: [2023-04-29 09:45:11,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +36: [2023-04-29 09:45:11,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:11,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... + 7: [2023-04-29 09:45:11,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +50: [2023-04-29 09:45:11,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +10: [2023-04-29 09:45:11,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +10: [2023-04-29 09:45:11,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +10: [2023-04-29 09:45:11,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +10: [2023-04-29 09:45:11,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +51: [2023-04-29 09:45:11,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +24: [2023-04-29 09:45:11,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:11,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +44: [2023-04-29 09:45:11,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +58: [2023-04-29 09:45:11,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +16: [2023-04-29 09:45:11,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +47: [2023-04-29 09:45:11,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:11,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +52: [2023-04-29 09:45:11,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:11,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +50: [2023-04-29 09:45:11,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +58: [2023-04-29 09:45:11,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:11,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +14: [2023-04-29 09:45:11,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +52: [2023-04-29 09:45:11,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +20: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +40: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +16: [2023-04-29 09:45:11,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +40: [2023-04-29 09:45:11,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +55: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +55: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +55: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +55: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +56: [2023-04-29 09:45:11,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +37: [2023-04-29 09:45:11,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:11,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +58: [2023-04-29 09:45:11,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +49: [2023-04-29 09:45:11,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:11,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +40: [2023-04-29 09:45:11,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +40: [2023-04-29 09:45:11,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +56: [2023-04-29 09:45:11,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +56: [2023-04-29 09:45:11,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +63: [2023-04-29 09:45:11,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +44: [2023-04-29 09:45:11,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +37: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +37: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +63: [2023-04-29 09:45:11,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +58: [2023-04-29 09:45:11,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:11,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:11,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +32: [2023-04-29 09:45:11,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +60: [2023-04-29 09:45:11,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +63: [2023-04-29 09:45:11,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +60: [2023-04-29 09:45:11,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +59: [2023-04-29 09:45:11,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +10: [2023-04-29 09:45:11,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +60: [2023-04-29 09:45:11,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +16: [2023-04-29 09:45:11,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +58: [2023-04-29 09:45:11,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +58: [2023-04-29 09:45:11,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +20: [2023-04-29 09:45:11,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:11,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +46: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +46: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +46: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +44: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +10: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 2: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 2: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +50: [2023-04-29 09:45:11,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +50: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +59: [2023-04-29 09:45:11,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +60: [2023-04-29 09:45:11,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 7: [2023-04-29 09:45:11,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:11,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +62: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +62: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +62: [2023-04-29 09:45:11,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +62: [2023-04-29 09:45:11,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +14: [2023-04-29 09:45:11,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:11,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:11,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +44: [2023-04-29 09:45:11,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +16: [2023-04-29 09:45:11,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +37: [2023-04-29 09:45:11,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:11,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +10: [2023-04-29 09:45:11,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +37: [2023-04-29 09:45:11,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:11,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +20: [2023-04-29 09:45:11,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:11,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +43: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 1: [2023-04-29 09:45:11,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +43: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +43: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +60: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +60: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +43: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 4: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +16: [2023-04-29 09:45:11,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +16: [2023-04-29 09:45:11,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:11,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +10: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +10: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +10: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +34: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +34: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +17: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +49: [2023-04-29 09:45:11,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +55: [2023-04-29 09:45:11,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +17: [2023-04-29 09:45:11,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +17: [2023-04-29 09:45:11,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +37: [2023-04-29 09:45:11,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +55: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +34: [2023-04-29 09:45:11,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +34: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +38: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +38: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +38: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +60: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +55: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +60: [2023-04-29 09:45:11,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_35-model_01-model_states.pt. +10: [2023-04-29 09:45:11,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +29: [2023-04-29 09:45:11,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +49: [2023-04-29 09:45:11,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +10: [2023-04-29 09:45:11,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +10: [2023-04-29 09:45:11,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:11,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +49: [2023-04-29 09:45:11,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +57: [2023-04-29 09:45:11,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +57: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +57: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +57: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +57: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +57: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +57: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +50: [2023-04-29 09:45:11,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 4: [2023-04-29 09:45:11,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +49: [2023-04-29 09:45:11,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +50: [2023-04-29 09:45:11,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +24: [2023-04-29 09:45:11,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +46: [2023-04-29 09:45:11,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +50: [2023-04-29 09:45:11,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +39: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +39: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +39: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +50: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +46: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +59: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:11,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +46: [2023-04-29 09:45:11,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +24: [2023-04-29 09:45:11,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +49: [2023-04-29 09:45:11,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +16: [2023-04-29 09:45:11,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +44: [2023-04-29 09:45:11,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +44: [2023-04-29 09:45:11,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:11,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +28: [2023-04-29 09:45:11,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +28: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +29: [2023-04-29 09:45:11,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +62: [2023-04-29 09:45:11,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +29: [2023-04-29 09:45:11,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +14: [2023-04-29 09:45:11,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +14: [2023-04-29 09:45:11,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +14: [2023-04-29 09:45:11,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +43: [2023-04-29 09:45:11,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +24: [2023-04-29 09:45:11,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +29: [2023-04-29 09:45:11,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +62: [2023-04-29 09:45:11,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +49: [2023-04-29 09:45:11,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +28: [2023-04-29 09:45:11,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +62: [2023-04-29 09:45:11,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +28: [2023-04-29 09:45:11,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:11,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:11,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:11,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +28: [2023-04-29 09:45:11,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +34: [2023-04-29 09:45:11,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +60: [2023-04-29 09:45:11,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +17: [2023-04-29 09:45:11,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:11,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:11,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 1: [2023-04-29 09:45:11,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +37: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +17: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +43: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +29: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +43: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +28: [2023-04-29 09:45:11,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +28: [2023-04-29 09:45:11,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +43: [2023-04-29 09:45:11,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +28: [2023-04-29 09:45:11,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +28: [2023-04-29 09:45:11,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +60: [2023-04-29 09:45:11,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +28: [2023-04-29 09:45:11,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +45: [2023-04-29 09:45:11,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +17: [2023-04-29 09:45:11,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +48: [2023-04-29 09:45:11,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:11,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:11,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +48: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +35: [2023-04-29 09:45:11,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +35: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:11,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +45: [2023-04-29 09:45:11,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:11,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:11,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +39: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +39: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +60: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +48: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +60: [2023-04-29 09:45:11,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +40: [2023-04-29 09:45:11,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +14: [2023-04-29 09:45:11,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +39: [2023-04-29 09:45:11,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +50: [2023-04-29 09:45:11,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:11,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:11,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:11,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +35: [2023-04-29 09:45:11,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +41: [2023-04-29 09:45:11,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:11,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +44: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +35: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +22: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +22: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +53: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +53: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +53: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +22: [2023-04-29 09:45:11,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +44: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +37: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:11,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +41: [2023-04-29 09:45:11,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +40: [2023-04-29 09:45:11,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:11,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +10: [2023-04-29 09:45:11,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +53: [2023-04-29 09:45:11,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +26: [2023-04-29 09:45:11,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +35: [2023-04-29 09:45:11,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +35: [2023-04-29 09:45:11,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +44: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +56: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +29: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +37: [2023-04-29 09:45:11,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:11,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +61: [2023-04-29 09:45:11,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +45: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +10: [2023-04-29 09:45:11,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +61: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +56: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +10: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +22: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +61: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +61: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +29: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +29: [2023-04-29 09:45:11,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +37: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +56: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +61: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +40: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +45: [2023-04-29 09:45:11,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +61: [2023-04-29 09:45:11,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +26: [2023-04-29 09:45:11,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +61: [2023-04-29 09:45:11,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +56: [2023-04-29 09:45:11,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +56: [2023-04-29 09:45:11,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +37: [2023-04-29 09:45:11,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:11,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:11,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:11,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +14: [2023-04-29 09:45:11,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:11,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +56: [2023-04-29 09:45:11,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +14: [2023-04-29 09:45:11,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:11,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +56: [2023-04-29 09:45:11,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +15: [2023-04-29 09:45:11,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +15: [2023-04-29 09:45:11,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +15: [2023-04-29 09:45:11,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +56: [2023-04-29 09:45:11,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +15: [2023-04-29 09:45:11,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +56: [2023-04-29 09:45:11,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:11,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +37: [2023-04-29 09:45:11,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +42: [2023-04-29 09:45:11,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +42: [2023-04-29 09:45:11,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +42: [2023-04-29 09:45:11,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +42: [2023-04-29 09:45:11,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +29: [2023-04-29 09:45:11,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:11,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +40: [2023-04-29 09:45:11,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:11,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:11,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +10: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +12: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +12: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +12: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +53: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:11,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +57: [2023-04-29 09:45:11,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +57: [2023-04-29 09:45:11,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +41: [2023-04-29 09:45:11,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:11,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:11,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +14: [2023-04-29 09:45:11,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +39: [2023-04-29 09:45:11,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:11,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:11,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +12: [2023-04-29 09:45:11,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +37: [2023-04-29 09:45:11,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:11,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:11,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:11,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:11,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +39: [2023-04-29 09:45:11,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:11,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:11,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +10: [2023-04-29 09:45:11,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +12: [2023-04-29 09:45:11,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +57: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +40: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +39: [2023-04-29 09:45:11,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:11,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:11,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 1: [2023-04-29 09:45:11,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +14: [2023-04-29 09:45:11,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +63: [2023-04-29 09:45:11,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +60: [2023-04-29 09:45:11,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +63: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +23: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +40: [2023-04-29 09:45:11,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +28: [2023-04-29 09:45:11,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +42: [2023-04-29 09:45:11,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:11,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +63: [2023-04-29 09:45:11,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +63: [2023-04-29 09:45:11,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +14: [2023-04-29 09:45:11,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +57: [2023-04-29 09:45:11,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +36: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +36: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +63: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +38: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:11,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +36: [2023-04-29 09:45:11,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +28: [2023-04-29 09:45:11,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +36: [2023-04-29 09:45:11,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +15: [2023-04-29 09:45:11,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +15: [2023-04-29 09:45:11,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +38: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +15: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +15: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +42: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +11: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +10: [2023-04-29 09:45:11,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +42: [2023-04-29 09:45:11,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:11,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:11,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +11: [2023-04-29 09:45:11,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +48: [2023-04-29 09:45:11,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:11,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +38: [2023-04-29 09:45:11,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +54: [2023-04-29 09:45:11,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +10: [2023-04-29 09:45:11,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:11,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +57: [2023-04-29 09:45:11,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +57: [2023-04-29 09:45:11,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:11,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +33: [2023-04-29 09:45:11,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +33: [2023-04-29 09:45:11,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 6: [2023-04-29 09:45:11,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +33: [2023-04-29 09:45:11,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +60: [2023-04-29 09:45:11,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +60: [2023-04-29 09:45:11,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:11,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +37: [2023-04-29 09:45:11,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +35: [2023-04-29 09:45:11,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +48: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +54: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +53: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:11,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +38: [2023-04-29 09:45:11,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +28: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +48: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:11,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +28: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +38: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +10: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +57: [2023-04-29 09:45:11,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +54: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +54: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +37: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +23: [2023-04-29 09:45:11,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +53: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:11,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +23: [2023-04-29 09:45:11,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +28: [2023-04-29 09:45:11,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +53: [2023-04-29 09:45:11,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +28: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +34: [2023-04-29 09:45:11,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +55: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +46: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +38: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +23: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +46: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +34: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +38: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +60: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +60: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +34: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +61: [2023-04-29 09:45:11,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +23: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +57: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +23: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +56: [2023-04-29 09:45:11,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:11,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:11,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +39: [2023-04-29 09:45:11,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +56: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +39: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +23: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +46: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +23: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +39: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +46: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +61: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +29: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +55: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +55: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +46: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +46: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +46: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +20: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +20: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +46: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +55: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +55: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +46: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +10: [2023-04-29 09:45:11,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +36: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +60: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +36: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +11: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +11: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +10: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:11,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +46: [2023-04-29 09:45:11,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +20: [2023-04-29 09:45:11,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +61: [2023-04-29 09:45:11,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +19: [2023-04-29 09:45:11,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +19: [2023-04-29 09:45:11,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +29: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +29: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +29: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +48: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +22: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +29: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +39: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +60: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +19: [2023-04-29 09:45:11,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +19: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +56: [2023-04-29 09:45:11,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +60: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +35: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +29: [2023-04-29 09:45:11,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +48: [2023-04-29 09:45:11,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +47: [2023-04-29 09:45:11,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +47: [2023-04-29 09:45:11,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:11,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:11,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +12: [2023-04-29 09:45:11,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +33: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +48: [2023-04-29 09:45:11,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +47: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +33: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +47: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +47: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +62: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +48: [2023-04-29 09:45:11,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:11,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +33: [2023-04-29 09:45:11,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:11,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +22: [2023-04-29 09:45:11,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +52: [2023-04-29 09:45:11,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +62: [2023-04-29 09:45:11,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +62: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +52: [2023-04-29 09:45:11,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +52: [2023-04-29 09:45:11,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +52: [2023-04-29 09:45:11,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +56: [2023-04-29 09:45:11,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +35: [2023-04-29 09:45:11,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +61: [2023-04-29 09:45:11,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +56: [2023-04-29 09:45:11,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:11,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +28: [2023-04-29 09:45:11,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +28: [2023-04-29 09:45:11,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +22: [2023-04-29 09:45:11,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +62: [2023-04-29 09:45:11,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +12: [2023-04-29 09:45:11,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +62: [2023-04-29 09:45:11,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +39: [2023-04-29 09:45:11,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:11,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +22: [2023-04-29 09:45:11,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:11,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:11,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:11,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +12: [2023-04-29 09:45:11,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +12: [2023-04-29 09:45:11,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +39: [2023-04-29 09:45:11,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:11,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +61: [2023-04-29 09:45:11,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:11,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +12: [2023-04-29 09:45:11,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 9: [2023-04-29 09:45:11,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +62: [2023-04-29 09:45:11,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +39: [2023-04-29 09:45:11,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:11,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +62: [2023-04-29 09:45:11,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt... +20: [2023-04-29 09:45:11,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +62: [2023-04-29 09:45:11,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt... +39: [2023-04-29 09:45:11,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +19: [2023-04-29 09:45:11,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +53: [2023-04-29 09:45:11,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +63: [2023-04-29 09:45:11,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +53: [2023-04-29 09:45:11,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +19: [2023-04-29 09:45:11,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +53: [2023-04-29 09:45:11,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +19: [2023-04-29 09:45:11,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:11,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +53: [2023-04-29 09:45:11,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +20: [2023-04-29 09:45:11,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +63: [2023-04-29 09:45:11,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +61: [2023-04-29 09:45:11,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +61: [2023-04-29 09:45:11,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:11,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +50: [2023-04-29 09:45:11,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +50: [2023-04-29 09:45:11,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +50: [2023-04-29 09:45:11,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +56: [2023-04-29 09:45:11,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +20: [2023-04-29 09:45:11,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +38: [2023-04-29 09:45:11,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 8: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 8: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 8: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +35: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +22: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +38: [2023-04-29 09:45:11,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +38: [2023-04-29 09:45:11,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:11,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 6: [2023-04-29 09:45:11,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +63: [2023-04-29 09:45:11,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 5: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +41: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:11,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +22: [2023-04-29 09:45:11,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:11,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +12: [2023-04-29 09:45:11,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +41: [2023-04-29 09:45:11,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +41: [2023-04-29 09:45:11,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +34: [2023-04-29 09:45:11,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:11,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +23: [2023-04-29 09:45:11,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +41: [2023-04-29 09:45:11,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +17: [2023-04-29 09:45:11,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +17: [2023-04-29 09:45:11,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +22: [2023-04-29 09:45:11,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +39: [2023-04-29 09:45:11,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:11,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +41: [2023-04-29 09:45:11,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +41: [2023-04-29 09:45:11,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +15: [2023-04-29 09:45:11,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +39: [2023-04-29 09:45:11,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:11,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +63: [2023-04-29 09:45:11,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +12: [2023-04-29 09:45:11,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +17: [2023-04-29 09:45:11,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +15: [2023-04-29 09:45:11,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:11,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +12: [2023-04-29 09:45:11,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:11,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +39: [2023-04-29 09:45:11,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:11,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:11,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:11,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:11,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 9: [2023-04-29 09:45:11,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:11,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +15: [2023-04-29 09:45:11,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +53: [2023-04-29 09:45:11,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +15: [2023-04-29 09:45:11,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +15: [2023-04-29 09:45:11,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +15: [2023-04-29 09:45:11,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +15: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +34: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +15: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +63: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +15: [2023-04-29 09:45:11,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +50: [2023-04-29 09:45:11,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:11,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +25: [2023-04-29 09:45:11,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +25: [2023-04-29 09:45:11,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +54: [2023-04-29 09:45:11,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +23: [2023-04-29 09:45:11,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +25: [2023-04-29 09:45:11,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +53: [2023-04-29 09:45:11,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:11,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:11,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +23: [2023-04-29 09:45:11,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +23: [2023-04-29 09:45:11,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +46: [2023-04-29 09:45:11,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:11,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +47: [2023-04-29 09:45:11,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +51: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 8: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +51: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 6: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +46: [2023-04-29 09:45:11,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +55: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +55: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. + 5: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:11,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +38: [2023-04-29 09:45:11,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:11,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +26: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +32: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 6: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +26: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +32: [2023-04-29 09:45:11,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +32: [2023-04-29 09:45:11,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +32: [2023-04-29 09:45:11,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +23: [2023-04-29 09:45:11,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +51: [2023-04-29 09:45:11,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:11,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:11,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +47: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +32: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +32: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +34: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +27: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +27: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +47: [2023-04-29 09:45:11,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +27: [2023-04-29 09:45:11,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 5: [2023-04-29 09:45:11,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:11,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 5: [2023-04-29 09:45:11,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:11,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:11,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:11,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +23: [2023-04-29 09:45:11,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:11,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +21: [2023-04-29 09:45:11,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +21: [2023-04-29 09:45:11,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +21: [2023-04-29 09:45:11,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +52: [2023-04-29 09:45:11,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +23: [2023-04-29 09:45:11,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +26: [2023-04-29 09:45:11,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 3: [2023-04-29 09:45:11,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 3: [2023-04-29 09:45:11,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +17: [2023-04-29 09:45:11,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:11,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +26: [2023-04-29 09:45:11,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +46: [2023-04-29 09:45:11,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:11,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +52: [2023-04-29 09:45:11,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +30: [2023-04-29 09:45:11,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +30: [2023-04-29 09:45:11,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +30: [2023-04-29 09:45:11,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +53: [2023-04-29 09:45:11,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +52: [2023-04-29 09:45:11,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +17: [2023-04-29 09:45:11,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +17: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +53: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +18: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +18: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +18: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +20: [2023-04-29 09:45:11,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +62: [2023-04-29 09:45:11,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +34: [2023-04-29 09:45:11,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:11,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:11,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +55: [2023-04-29 09:45:11,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +25: [2023-04-29 09:45:11,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +34: [2023-04-29 09:45:11,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +23: [2023-04-29 09:45:11,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +52: [2023-04-29 09:45:11,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +54: [2023-04-29 09:45:11,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:11,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +25: [2023-04-29 09:45:11,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +25: [2023-04-29 09:45:11,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +23: [2023-04-29 09:45:11,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +34: [2023-04-29 09:45:11,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:11,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +62: [2023-04-29 09:45:11,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_00-model_states.pt. +46: [2023-04-29 09:45:11,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:11,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:11,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +25: [2023-04-29 09:45:11,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:11,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +25: [2023-04-29 09:45:11,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +54: [2023-04-29 09:45:11,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +25: [2023-04-29 09:45:11,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +44: [2023-04-29 09:45:11,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +44: [2023-04-29 09:45:11,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +44: [2023-04-29 09:45:11,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +25: [2023-04-29 09:45:11,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +25: [2023-04-29 09:45:11,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +44: [2023-04-29 09:45:11,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +25: [2023-04-29 09:45:11,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +55: [2023-04-29 09:45:11,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +27: [2023-04-29 09:45:11,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 0: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +47: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +55: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:11,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +27: [2023-04-29 09:45:11,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +47: [2023-04-29 09:45:11,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:11,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +20: [2023-04-29 09:45:11,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:11,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +20: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +23: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +62: [2023-04-29 09:45:11,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:11,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 3: [2023-04-29 09:45:11,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:11,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:11,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +23: [2023-04-29 09:45:11,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:11,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:11,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:11,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +52: [2023-04-29 09:45:11,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +43: [2023-04-29 09:45:11,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +43: [2023-04-29 09:45:11,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +43: [2023-04-29 09:45:11,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:11,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:11,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +20: [2023-04-29 09:45:11,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +62: [2023-04-29 09:45:11,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +18: [2023-04-29 09:45:11,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +52: [2023-04-29 09:45:11,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +20: [2023-04-29 09:45:11,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +22: [2023-04-29 09:45:11,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +20: [2023-04-29 09:45:11,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +42: [2023-04-29 09:45:11,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:11,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +42: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +22: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +43: [2023-04-29 09:45:11,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +43: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 7: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +43: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 7: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +18: [2023-04-29 09:45:12,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +52: [2023-04-29 09:45:12,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +18: [2023-04-29 09:45:12,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +22: [2023-04-29 09:45:12,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +22: [2023-04-29 09:45:12,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +42: [2023-04-29 09:45:12,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +42: [2023-04-29 09:45:12,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +12: [2023-04-29 09:45:12,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:12,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +42: [2023-04-29 09:45:12,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:12,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +42: [2023-04-29 09:45:12,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:12,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:12,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +22: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +15: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +41: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +13: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +13: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +13: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +13: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +41: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:12,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:12,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +12: [2023-04-29 09:45:12,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +42: [2023-04-29 09:45:12,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +42: [2023-04-29 09:45:12,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +42: [2023-04-29 09:45:12,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +44: [2023-04-29 09:45:12,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +42: [2023-04-29 09:45:12,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:12,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +59: [2023-04-29 09:45:12,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +12: [2023-04-29 09:45:12,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +12: [2023-04-29 09:45:12,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +59: [2023-04-29 09:45:12,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +59: [2023-04-29 09:45:12,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +15: [2023-04-29 09:45:12,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +62: [2023-04-29 09:45:12,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +32: [2023-04-29 09:45:12,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +12: [2023-04-29 09:45:12,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +12: [2023-04-29 09:45:12,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +26: [2023-04-29 09:45:12,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:12,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +51: [2023-04-29 09:45:12,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +32: [2023-04-29 09:45:12,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +40: [2023-04-29 09:45:12,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +58: [2023-04-29 09:45:12,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:12,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:12,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:12,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:12,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 2: [2023-04-29 09:45:12,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:12,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +40: [2023-04-29 09:45:12,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +40: [2023-04-29 09:45:12,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 2: [2023-04-29 09:45:12,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 2: [2023-04-29 09:45:12,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +41: [2023-04-29 09:45:12,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +32: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +26: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:12,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +26: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +17: [2023-04-29 09:45:12,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +33: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +51: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +51: [2023-04-29 09:45:12,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +33: [2023-04-29 09:45:12,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:12,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +17: [2023-04-29 09:45:12,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +41: [2023-04-29 09:45:12,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +15: [2023-04-29 09:45:12,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +33: [2023-04-29 09:45:12,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:12,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +33: [2023-04-29 09:45:12,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:12,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:12,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +13: [2023-04-29 09:45:12,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:12,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:12,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:12,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +57: [2023-04-29 09:45:12,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +57: [2023-04-29 09:45:12,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +59: [2023-04-29 09:45:12,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +57: [2023-04-29 09:45:12,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +25: [2023-04-29 09:45:12,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +57: [2023-04-29 09:45:12,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +33: [2023-04-29 09:45:12,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +15: [2023-04-29 09:45:12,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +32: [2023-04-29 09:45:12,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +13: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +51: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +13: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +33: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +33: [2023-04-29 09:45:12,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +11: [2023-04-29 09:45:12,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +15: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +25: [2023-04-29 09:45:12,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:12,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:12,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +25: [2023-04-29 09:45:12,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +21: [2023-04-29 09:45:12,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:12,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +40: [2023-04-29 09:45:12,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:12,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +11: [2023-04-29 09:45:12,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +13: [2023-04-29 09:45:12,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:12,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:12,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:12,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +43: [2023-04-29 09:45:12,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:12,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +25: [2023-04-29 09:45:12,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +59: [2023-04-29 09:45:12,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +11: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +11: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +17: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +40: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +17: [2023-04-29 09:45:12,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +11: [2023-04-29 09:45:12,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +11: [2023-04-29 09:45:12,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:12,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:12,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:12,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +21: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +49: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +49: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +49: [2023-04-29 09:45:12,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:12,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +51: [2023-04-29 09:45:12,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +26: [2023-04-29 09:45:12,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +32: [2023-04-29 09:45:12,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +40: [2023-04-29 09:45:12,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +17: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +31: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +31: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +30: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +32: [2023-04-29 09:45:12,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +30: [2023-04-29 09:45:12,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:12,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +26: [2023-04-29 09:45:12,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +51: [2023-04-29 09:45:12,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +31: [2023-04-29 09:45:12,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +32: [2023-04-29 09:45:12,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_14-model_01-model_states.pt. +26: [2023-04-29 09:45:12,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:12,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +43: [2023-04-29 09:45:12,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +21: [2023-04-29 09:45:12,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +51: [2023-04-29 09:45:12,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +21: [2023-04-29 09:45:12,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +42: [2023-04-29 09:45:12,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:12,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +25: [2023-04-29 09:45:12,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +25: [2023-04-29 09:45:12,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +25: [2023-04-29 09:45:12,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +57: [2023-04-29 09:45:12,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +57: [2023-04-29 09:45:12,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:12,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +21: [2023-04-29 09:45:12,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +21: [2023-04-29 09:45:12,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +51: [2023-04-29 09:45:12,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +57: [2023-04-29 09:45:12,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +51: [2023-04-29 09:45:12,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +21: [2023-04-29 09:45:12,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +30: [2023-04-29 09:45:12,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +57: [2023-04-29 09:45:12,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +18: [2023-04-29 09:45:12,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +43: [2023-04-29 09:45:12,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +43: [2023-04-29 09:45:12,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:12,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:12,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +30: [2023-04-29 09:45:12,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +25: [2023-04-29 09:45:12,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +43: [2023-04-29 09:45:12,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +25: [2023-04-29 09:45:12,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +25: [2023-04-29 09:45:12,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:12,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +18: [2023-04-29 09:45:12,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +58: [2023-04-29 09:45:12,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +56: [2023-04-29 09:45:12,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +56: [2023-04-29 09:45:12,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +18: [2023-04-29 09:45:12,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +56: [2023-04-29 09:45:12,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +25: [2023-04-29 09:45:12,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +30: [2023-04-29 09:45:12,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +25: [2023-04-29 09:45:12,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +42: [2023-04-29 09:45:12,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +18: [2023-04-29 09:45:12,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +35: [2023-04-29 09:45:12,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +35: [2023-04-29 09:45:12,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +35: [2023-04-29 09:45:12,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +35: [2023-04-29 09:45:12,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +18: [2023-04-29 09:45:12,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +58: [2023-04-29 09:45:12,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +42: [2023-04-29 09:45:12,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +58: [2023-04-29 09:45:12,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +31: [2023-04-29 09:45:12,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +33: [2023-04-29 09:45:12,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +31: [2023-04-29 09:45:12,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +31: [2023-04-29 09:45:12,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +31: [2023-04-29 09:45:12,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +36: [2023-04-29 09:45:12,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:12,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +36: [2023-04-29 09:45:12,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +36: [2023-04-29 09:45:12,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +61: [2023-04-29 09:45:12,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +61: [2023-04-29 09:45:12,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +61: [2023-04-29 09:45:12,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +61: [2023-04-29 09:45:12,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +36: [2023-04-29 09:45:12,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +36: [2023-04-29 09:45:12,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:12,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +36: [2023-04-29 09:45:12,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +42: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +43: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +43: [2023-04-29 09:45:12,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +13: [2023-04-29 09:45:12,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +13: [2023-04-29 09:45:12,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +31: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +33: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 7: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +59: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +27: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +56: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +31: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +56: [2023-04-29 09:45:12,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +27: [2023-04-29 09:45:12,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +33: [2023-04-29 09:45:12,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +25: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +33: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +56: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +24: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +59: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +24: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +24: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +21: [2023-04-29 09:45:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +13: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +13: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +27: [2023-04-29 09:45:12,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +24: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +11: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +42: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +27: [2023-04-29 09:45:12,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +59: [2023-04-29 09:45:12,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:12,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:12,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:12,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +33: [2023-04-29 09:45:12,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +19: [2023-04-29 09:45:12,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +19: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +59: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +27: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:12,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +25: [2023-04-29 09:45:12,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +27: [2023-04-29 09:45:12,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +27: [2023-04-29 09:45:12,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +59: [2023-04-29 09:45:12,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +21: [2023-04-29 09:45:12,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +59: [2023-04-29 09:45:12,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +59: [2023-04-29 09:45:12,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +30: [2023-04-29 09:45:12,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +58: [2023-04-29 09:45:12,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +19: [2023-04-29 09:45:12,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +19: [2023-04-29 09:45:12,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 2: [2023-04-29 09:45:12,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +13: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +58: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +63: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +63: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +63: [2023-04-29 09:45:12,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +63: [2023-04-29 09:45:12,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +35: [2023-04-29 09:45:12,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +13: [2023-04-29 09:45:12,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +35: [2023-04-29 09:45:12,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +18: [2023-04-29 09:45:12,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +61: [2023-04-29 09:45:12,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +61: [2023-04-29 09:45:12,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:12,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 4: [2023-04-29 09:45:12,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 4: [2023-04-29 09:45:12,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 4: [2023-04-29 09:45:12,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +61: [2023-04-29 09:45:12,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +19: [2023-04-29 09:45:12,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +18: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +61: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +30: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +30: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:12,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +19: [2023-04-29 09:45:12,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +19: [2023-04-29 09:45:12,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +19: [2023-04-29 09:45:12,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +21: [2023-04-29 09:45:12,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:12,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +51: [2023-04-29 09:45:12,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +18: [2023-04-29 09:45:12,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +59: [2023-04-29 09:45:12,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 5: [2023-04-29 09:45:12,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 5: [2023-04-29 09:45:12,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 2: [2023-04-29 09:45:12,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +33: [2023-04-29 09:45:12,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 5: [2023-04-29 09:45:12,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +51: [2023-04-29 09:45:12,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +33: [2023-04-29 09:45:12,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:12,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:12,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +51: [2023-04-29 09:45:12,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +24: [2023-04-29 09:45:12,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +21: [2023-04-29 09:45:12,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +11: [2023-04-29 09:45:12,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +33: [2023-04-29 09:45:12,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +11: [2023-04-29 09:45:12,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +51: [2023-04-29 09:45:12,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +51: [2023-04-29 09:45:12,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +24: [2023-04-29 09:45:12,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +63: [2023-04-29 09:45:12,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +24: [2023-04-29 09:45:12,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +51: [2023-04-29 09:45:12,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +30: [2023-04-29 09:45:12,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +63: [2023-04-29 09:45:12,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:12,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:12,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +60: [2023-04-29 09:45:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +63: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +60: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 9: [2023-04-29 09:45:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +18: [2023-04-29 09:45:12,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +18: [2023-04-29 09:45:12,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +47: [2023-04-29 09:45:12,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +47: [2023-04-29 09:45:12,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +47: [2023-04-29 09:45:12,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +47: [2023-04-29 09:45:12,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:12,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +21: [2023-04-29 09:45:12,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +45: [2023-04-29 09:45:12,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +31: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... +30: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +46: [2023-04-29 09:45:12,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +46: [2023-04-29 09:45:12,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:12,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +31: [2023-04-29 09:45:12,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +36: [2023-04-29 09:45:12,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:12,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +46: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +45: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +24: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +36: [2023-04-29 09:45:12,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +14: [2023-04-29 09:45:12,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +14: [2023-04-29 09:45:12,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +14: [2023-04-29 09:45:12,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +14: [2023-04-29 09:45:12,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +54: [2023-04-29 09:45:12,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +54: [2023-04-29 09:45:12,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +31: [2023-04-29 09:45:12,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +30: [2023-04-29 09:45:12,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:12,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +27: [2023-04-29 09:45:12,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +54: [2023-04-29 09:45:12,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 3: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +55: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +55: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +55: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +55: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 8: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +54: [2023-04-29 09:45:12,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 8: [2023-04-29 09:45:12,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +31: [2023-04-29 09:45:12,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt... +31: [2023-04-29 09:45:12,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +60: [2023-04-29 09:45:12,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +27: [2023-04-29 09:45:12,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +60: [2023-04-29 09:45:12,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:12,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +47: [2023-04-29 09:45:12,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +60: [2023-04-29 09:45:12,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:12,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +59: [2023-04-29 09:45:12,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +31: [2023-04-29 09:45:12,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +19: [2023-04-29 09:45:12,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +59: [2023-04-29 09:45:12,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +46: [2023-04-29 09:45:12,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:12,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:12,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:12,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +49: [2023-04-29 09:45:12,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +49: [2023-04-29 09:45:12,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +49: [2023-04-29 09:45:12,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +46: [2023-04-29 09:45:12,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:12,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +52: [2023-04-29 09:45:12,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +52: [2023-04-29 09:45:12,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +49: [2023-04-29 09:45:12,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +52: [2023-04-29 09:45:12,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +49: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +52: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +49: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +54: [2023-04-29 09:45:12,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:12,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +14: [2023-04-29 09:45:12,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +19: [2023-04-29 09:45:12,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +62: [2023-04-29 09:45:12,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +62: [2023-04-29 09:45:12,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +14: [2023-04-29 09:45:12,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +27: [2023-04-29 09:45:12,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +62: [2023-04-29 09:45:12,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +50: [2023-04-29 09:45:12,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:12,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:12,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +62: [2023-04-29 09:45:12,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_36-model_01-model_states.pt. +31: [2023-04-29 09:45:12,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +27: [2023-04-29 09:45:12,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +55: [2023-04-29 09:45:12,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +27: [2023-04-29 09:45:12,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +50: [2023-04-29 09:45:12,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +50: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +54: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +55: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +55: [2023-04-29 09:45:12,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +55: [2023-04-29 09:45:12,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +19: [2023-04-29 09:45:12,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +50: [2023-04-29 09:45:12,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +59: [2023-04-29 09:45:12,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:12,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +50: [2023-04-29 09:45:12,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +19: [2023-04-29 09:45:12,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:12,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +27: [2023-04-29 09:45:12,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +19: [2023-04-29 09:45:12,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +52: [2023-04-29 09:45:12,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +52: [2023-04-29 09:45:12,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +45: [2023-04-29 09:45:12,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +52: [2023-04-29 09:45:12,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +52: [2023-04-29 09:45:12,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_00-model_states.pt. +24: [2023-04-29 09:45:12,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +62: [2023-04-29 09:45:12,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:12,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +62: [2023-04-29 09:45:12,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:12,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +24: [2023-04-29 09:45:12,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +45: [2023-04-29 09:45:12,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +45: [2023-04-29 09:45:12,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +19: [2023-04-29 09:45:12,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +35: [2023-04-29 09:45:12,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +19: [2023-04-29 09:45:12,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:12,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:12,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +17: [2023-04-29 09:45:12,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +35: [2023-04-29 09:45:12,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:12,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +35: [2023-04-29 09:45:12,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:12,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +28: [2023-04-29 09:45:12,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +28: [2023-04-29 09:45:12,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +28: [2023-04-29 09:45:12,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +45: [2023-04-29 09:45:12,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +28: [2023-04-29 09:45:12,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +17: [2023-04-29 09:45:12,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 1: [2023-04-29 09:45:12,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 1: [2023-04-29 09:45:12,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 1: [2023-04-29 09:45:12,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 9: [2023-04-29 09:45:12,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:12,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +58: [2023-04-29 09:45:12,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +58: [2023-04-29 09:45:12,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +58: [2023-04-29 09:45:12,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +45: [2023-04-29 09:45:12,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:12,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +38: [2023-04-29 09:45:12,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +45: [2023-04-29 09:45:12,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:12,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +20: [2023-04-29 09:45:12,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +49: [2023-04-29 09:45:12,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:12,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:12,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +20: [2023-04-29 09:45:12,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +34: [2023-04-29 09:45:12,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +34: [2023-04-29 09:45:12,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +37: [2023-04-29 09:45:12,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +37: [2023-04-29 09:45:12,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +39: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +39: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +39: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +39: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +34: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +34: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +37: [2023-04-29 09:45:12,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +37: [2023-04-29 09:45:12,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +20: [2023-04-29 09:45:12,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +20: [2023-04-29 09:45:12,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +49: [2023-04-29 09:45:12,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +40: [2023-04-29 09:45:12,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:12,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +28: [2023-04-29 09:45:12,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +40: [2023-04-29 09:45:12,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +28: [2023-04-29 09:45:12,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +40: [2023-04-29 09:45:12,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:12,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +40: [2023-04-29 09:45:12,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +58: [2023-04-29 09:45:12,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +38: [2023-04-29 09:45:12,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +38: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +45: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +24: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +45: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +45: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +28: [2023-04-29 09:45:12,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +58: [2023-04-29 09:45:12,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:12,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:12,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +39: [2023-04-29 09:45:12,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +39: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:12,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:12,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:12,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:12,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +44: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 1: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +44: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +28: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +28: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +35: [2023-04-29 09:45:12,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:12,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:12,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +34: [2023-04-29 09:45:12,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:12,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +34: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +28: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +48: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +28: [2023-04-29 09:45:12,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +57: [2023-04-29 09:45:12,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +35: [2023-04-29 09:45:12,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +50: [2023-04-29 09:45:12,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +17: [2023-04-29 09:45:12,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +57: [2023-04-29 09:45:12,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +57: [2023-04-29 09:45:12,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +57: [2023-04-29 09:45:12,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +28: [2023-04-29 09:45:12,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +57: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +57: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +57: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +57: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +28: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +57: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +56: [2023-04-29 09:45:12,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +56: [2023-04-29 09:45:12,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +56: [2023-04-29 09:45:12,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +56: [2023-04-29 09:45:12,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +45: [2023-04-29 09:45:12,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:12,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:12,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +35: [2023-04-29 09:45:12,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +17: [2023-04-29 09:45:12,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +56: [2023-04-29 09:45:12,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +56: [2023-04-29 09:45:12,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +45: [2023-04-29 09:45:12,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:12,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +23: [2023-04-29 09:45:12,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +23: [2023-04-29 09:45:12,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +23: [2023-04-29 09:45:12,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +23: [2023-04-29 09:45:12,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +37: [2023-04-29 09:45:12,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +32: [2023-04-29 09:45:12,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +20: [2023-04-29 09:45:12,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +20: [2023-04-29 09:45:12,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +20: [2023-04-29 09:45:12,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +35: [2023-04-29 09:45:12,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +20: [2023-04-29 09:45:12,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +32: [2023-04-29 09:45:12,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +32: [2023-04-29 09:45:12,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +17: [2023-04-29 09:45:12,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:12,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +35: [2023-04-29 09:45:12,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +17: [2023-04-29 09:45:12,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +40: [2023-04-29 09:45:12,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +35: [2023-04-29 09:45:12,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +35: [2023-04-29 09:45:12,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +17: [2023-04-29 09:45:12,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +17: [2023-04-29 09:45:12,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +44: [2023-04-29 09:45:12,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 1: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +58: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +23: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +23: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +43: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +43: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +43: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +43: [2023-04-29 09:45:12,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +20: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +20: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +32: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +20: [2023-04-29 09:45:12,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +58: [2023-04-29 09:45:12,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +40: [2023-04-29 09:45:12,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:12,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:12,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +58: [2023-04-29 09:45:12,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +29: [2023-04-29 09:45:12,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +58: [2023-04-29 09:45:12,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +29: [2023-04-29 09:45:12,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +29: [2023-04-29 09:45:12,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:12,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +58: [2023-04-29 09:45:12,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +48: [2023-04-29 09:45:12,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +20: [2023-04-29 09:45:12,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +28: [2023-04-29 09:45:12,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +57: [2023-04-29 09:45:12,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:12,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +37: [2023-04-29 09:45:12,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +53: [2023-04-29 09:45:12,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +53: [2023-04-29 09:45:12,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +53: [2023-04-29 09:45:12,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +48: [2023-04-29 09:45:12,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:12,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +20: [2023-04-29 09:45:12,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:12,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +56: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +41: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +41: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +57: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +37: [2023-04-29 09:45:12,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +51: [2023-04-29 09:45:12,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:12,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +41: [2023-04-29 09:45:12,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +43: [2023-04-29 09:45:12,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:12,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:12,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:12,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +37: [2023-04-29 09:45:12,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +58: [2023-04-29 09:45:12,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +28: [2023-04-29 09:45:12,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +51: [2023-04-29 09:45:12,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +56: [2023-04-29 09:45:12,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +44: [2023-04-29 09:45:12,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +51: [2023-04-29 09:45:12,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:12,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +58: [2023-04-29 09:45:12,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:12,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:12,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +45: [2023-04-29 09:45:12,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:12,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:12,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:12,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +40: [2023-04-29 09:45:12,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +57: [2023-04-29 09:45:12,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +63: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +12: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +12: [2023-04-29 09:45:12,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +12: [2023-04-29 09:45:12,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +28: [2023-04-29 09:45:12,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +28: [2023-04-29 09:45:12,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +57: [2023-04-29 09:45:12,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +45: [2023-04-29 09:45:12,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 2: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 2: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 2: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +63: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +63: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +22: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +63: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +48: [2023-04-29 09:45:12,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +45: [2023-04-29 09:45:12,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:12,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +29: [2023-04-29 09:45:12,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +29: [2023-04-29 09:45:12,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +29: [2023-04-29 09:45:12,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +63: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +63: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +22: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +10: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +10: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +22: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +10: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +29: [2023-04-29 09:45:12,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +48: [2023-04-29 09:45:12,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:12,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +20: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +45: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +63: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +63: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +62: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:12,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +60: [2023-04-29 09:45:12,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +45: [2023-04-29 09:45:12,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +37: [2023-04-29 09:45:12,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +63: [2023-04-29 09:45:12,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +62: [2023-04-29 09:45:12,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +62: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +62: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +28: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +28: [2023-04-29 09:45:12,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +57: [2023-04-29 09:45:12,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +41: [2023-04-29 09:45:12,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +37: [2023-04-29 09:45:12,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +60: [2023-04-29 09:45:12,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:12,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +57: [2023-04-29 09:45:12,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:12,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +62: [2023-04-29 09:45:12,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:12,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +41: [2023-04-29 09:45:12,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +60: [2023-04-29 09:45:12,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +62: [2023-04-29 09:45:12,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +60: [2023-04-29 09:45:12,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +62: [2023-04-29 09:45:12,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +56: [2023-04-29 09:45:12,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:12,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:12,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:12,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:12,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:12,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:12,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:12,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:12,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +54: [2023-04-29 09:45:12,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:12,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +56: [2023-04-29 09:45:12,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:12,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +55: [2023-04-29 09:45:12,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +10: [2023-04-29 09:45:12,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:12,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:12,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +55: [2023-04-29 09:45:12,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +22: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +55: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +55: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +55: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +29: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +55: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +55: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +55: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +54: [2023-04-29 09:45:12,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +54: [2023-04-29 09:45:12,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +29: [2023-04-29 09:45:12,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +22: [2023-04-29 09:45:12,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +42: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +42: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +29: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +10: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +54: [2023-04-29 09:45:12,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +54: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +14: [2023-04-29 09:45:12,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +54: [2023-04-29 09:45:12,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +54: [2023-04-29 09:45:12,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +14: [2023-04-29 09:45:12,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +51: [2023-04-29 09:45:12,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:12,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:12,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +28: [2023-04-29 09:45:12,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +28: [2023-04-29 09:45:12,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +37: [2023-04-29 09:45:12,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +37: [2023-04-29 09:45:12,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:12,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +29: [2023-04-29 09:45:12,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:12,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +14: [2023-04-29 09:45:12,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +14: [2023-04-29 09:45:12,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +18: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +18: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +18: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +18: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +14: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +14: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +29: [2023-04-29 09:45:12,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +27: [2023-04-29 09:45:12,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +29: [2023-04-29 09:45:12,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:12,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +27: [2023-04-29 09:45:12,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +27: [2023-04-29 09:45:12,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +27: [2023-04-29 09:45:12,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +20: [2023-04-29 09:45:12,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:12,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +33: [2023-04-29 09:45:12,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +33: [2023-04-29 09:45:12,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +33: [2023-04-29 09:45:12,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:12,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:12,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +34: [2023-04-29 09:45:12,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +20: [2023-04-29 09:45:12,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +34: [2023-04-29 09:45:12,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +34: [2023-04-29 09:45:12,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +36: [2023-04-29 09:45:12,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +36: [2023-04-29 09:45:12,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +42: [2023-04-29 09:45:12,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +51: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 8: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 8: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 8: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +42: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +34: [2023-04-29 09:45:12,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:12,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +20: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +38: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +38: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +38: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +38: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +38: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +16: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +16: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +16: [2023-04-29 09:45:12,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +38: [2023-04-29 09:45:12,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +16: [2023-04-29 09:45:12,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +38: [2023-04-29 09:45:12,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +20: [2023-04-29 09:45:12,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +26: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +62: [2023-04-29 09:45:12,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +63: [2023-04-29 09:45:12,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +61: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +53: [2023-04-29 09:45:12,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +20: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +60: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +27: [2023-04-29 09:45:12,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +11: [2023-04-29 09:45:12,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +25: [2023-04-29 09:45:12,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +25: [2023-04-29 09:45:12,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +18: [2023-04-29 09:45:12,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +61: [2023-04-29 09:45:12,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:12,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +61: [2023-04-29 09:45:12,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +11: [2023-04-29 09:45:12,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +53: [2023-04-29 09:45:12,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +61: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +53: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +60: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +25: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +39: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +39: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +39: [2023-04-29 09:45:12,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +25: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +39: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +39: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +39: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +39: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +29: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +42: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +42: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +27: [2023-04-29 09:45:12,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +15: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +61: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +15: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +15: [2023-04-29 09:45:12,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +61: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +15: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +61: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +18: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +41: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +18: [2023-04-29 09:45:12,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:12,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +53: [2023-04-29 09:45:12,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +33: [2023-04-29 09:45:12,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +18: [2023-04-29 09:45:12,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:12,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:12,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +58: [2023-04-29 09:45:12,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +27: [2023-04-29 09:45:12,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +58: [2023-04-29 09:45:12,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +53: [2023-04-29 09:45:12,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +51: [2023-04-29 09:45:12,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +53: [2023-04-29 09:45:12,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +41: [2023-04-29 09:45:12,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +41: [2023-04-29 09:45:12,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:12,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:12,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +29: [2023-04-29 09:45:12,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +16: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +33: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:12,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +32: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +46: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +46: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +12: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +60: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +58: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +16: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +41: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:12,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:12,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +26: [2023-04-29 09:45:12,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +32: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +36: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +45: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +62: [2023-04-29 09:45:12,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +53: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +16: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +45: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +46: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +46: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +46: [2023-04-29 09:45:12,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +46: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +46: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +45: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +12: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +16: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +32: [2023-04-29 09:45:12,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +53: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +12: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +32: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +26: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +45: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +26: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +36: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +36: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +36: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +36: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +12: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +26: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +32: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +12: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +32: [2023-04-29 09:45:12,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +58: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +46: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +32: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +63: [2023-04-29 09:45:12,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +11: [2023-04-29 09:45:12,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +41: [2023-04-29 09:45:12,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +21: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +21: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +46: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +46: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +11: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +63: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +21: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +22: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +46: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +60: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +63: [2023-04-29 09:45:12,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +11: [2023-04-29 09:45:12,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +60: [2023-04-29 09:45:12,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +36: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +55: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +22: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +49: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +49: [2023-04-29 09:45:12,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +22: [2023-04-29 09:45:12,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +25: [2023-04-29 09:45:12,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +63: [2023-04-29 09:45:12,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +22: [2023-04-29 09:45:12,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +25: [2023-04-29 09:45:12,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:12,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +22: [2023-04-29 09:45:12,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +29: [2023-04-29 09:45:12,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +54: [2023-04-29 09:45:12,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:12,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:12,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:12,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +22: [2023-04-29 09:45:12,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +14: [2023-04-29 09:45:12,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +54: [2023-04-29 09:45:12,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +22: [2023-04-29 09:45:12,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +22: [2023-04-29 09:45:12,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +55: [2023-04-29 09:45:12,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +22: [2023-04-29 09:45:12,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +29: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +29: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 6: [2023-04-29 09:45:12,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +62: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:12,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +59: [2023-04-29 09:45:12,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +59: [2023-04-29 09:45:12,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +10: [2023-04-29 09:45:12,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +34: [2023-04-29 09:45:12,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 6: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +29: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +14: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +42: [2023-04-29 09:45:12,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 3: [2023-04-29 09:45:12,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 3: [2023-04-29 09:45:12,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +13: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +13: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 3: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +38: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +13: [2023-04-29 09:45:12,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +53: [2023-04-29 09:45:12,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +62: [2023-04-29 09:45:12,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +13: [2023-04-29 09:45:12,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +53: [2023-04-29 09:45:12,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +10: [2023-04-29 09:45:12,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +29: [2023-04-29 09:45:12,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +38: [2023-04-29 09:45:12,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +30: [2023-04-29 09:45:12,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +30: [2023-04-29 09:45:12,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +53: [2023-04-29 09:45:12,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +30: [2023-04-29 09:45:12,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +10: [2023-04-29 09:45:12,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +10: [2023-04-29 09:45:12,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +19: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +19: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +42: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:12,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +55: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +55: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +55: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +10: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +47: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +47: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +10: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +34: [2023-04-29 09:45:12,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +19: [2023-04-29 09:45:12,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +19: [2023-04-29 09:45:12,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +10: [2023-04-29 09:45:12,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +42: [2023-04-29 09:45:12,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +42: [2023-04-29 09:45:12,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +43: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +52: [2023-04-29 09:45:12,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +42: [2023-04-29 09:45:12,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +47: [2023-04-29 09:45:12,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +54: [2023-04-29 09:45:12,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +39: [2023-04-29 09:45:12,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +47: [2023-04-29 09:45:12,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +47: [2023-04-29 09:45:12,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +43: [2023-04-29 09:45:12,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +47: [2023-04-29 09:45:12,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +47: [2023-04-29 09:45:12,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +61: [2023-04-29 09:45:12,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +39: [2023-04-29 09:45:12,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +43: [2023-04-29 09:45:12,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +42: [2023-04-29 09:45:12,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +52: [2023-04-29 09:45:12,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +52: [2023-04-29 09:45:12,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +52: [2023-04-29 09:45:12,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +52: [2023-04-29 09:45:12,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +61: [2023-04-29 09:45:12,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +61: [2023-04-29 09:45:12,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +52: [2023-04-29 09:45:12,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... +52: [2023-04-29 09:45:12,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt... +41: [2023-04-29 09:45:12,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:12,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +54: [2023-04-29 09:45:12,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +13: [2023-04-29 09:45:12,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 0: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +34: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:12,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +26: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +59: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +26: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +45: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:12,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +14: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +41: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +35: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +35: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +35: [2023-04-29 09:45:12,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 0: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +17: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +17: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 0: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +12: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +59: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +16: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +45: [2023-04-29 09:45:12,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +14: [2023-04-29 09:45:12,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +26: [2023-04-29 09:45:12,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +30: [2023-04-29 09:45:12,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +30: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +38: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +17: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +17: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +26: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +26: [2023-04-29 09:45:12,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +32: [2023-04-29 09:45:12,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +22: [2023-04-29 09:45:12,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +16: [2023-04-29 09:45:12,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +59: [2023-04-29 09:45:12,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +59: [2023-04-29 09:45:12,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +59: [2023-04-29 09:45:12,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +59: [2023-04-29 09:45:12,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +46: [2023-04-29 09:45:12,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +19: [2023-04-29 09:45:12,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +42: [2023-04-29 09:45:12,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +46: [2023-04-29 09:45:12,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +12: [2023-04-29 09:45:12,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +36: [2023-04-29 09:45:12,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +12: [2023-04-29 09:45:12,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +38: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +38: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +24: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +24: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +24: [2023-04-29 09:45:12,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +36: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +36: [2023-04-29 09:45:12,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +19: [2023-04-29 09:45:12,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +19: [2023-04-29 09:45:12,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +19: [2023-04-29 09:45:12,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +12: [2023-04-29 09:45:12,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +51: [2023-04-29 09:45:12,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +51: [2023-04-29 09:45:12,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +51: [2023-04-29 09:45:12,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +51: [2023-04-29 09:45:12,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +26: [2023-04-29 09:45:12,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +32: [2023-04-29 09:45:12,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +16: [2023-04-29 09:45:12,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +16: [2023-04-29 09:45:12,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +59: [2023-04-29 09:45:12,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +59: [2023-04-29 09:45:12,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +59: [2023-04-29 09:45:12,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +61: [2023-04-29 09:45:12,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +22: [2023-04-29 09:45:12,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:12,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:12,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:12,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:12,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +26: [2023-04-29 09:45:12,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:12,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +21: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +34: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +36: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +21: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +46: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +21: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +36: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +61: [2023-04-29 09:45:12,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +26: [2023-04-29 09:45:12,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +39: [2023-04-29 09:45:12,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +11: [2023-04-29 09:45:12,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +36: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +36: [2023-04-29 09:45:12,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +31: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +61: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +39: [2023-04-29 09:45:12,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +36: [2023-04-29 09:45:12,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:12,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +21: [2023-04-29 09:45:12,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +26: [2023-04-29 09:45:12,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +36: [2023-04-29 09:45:12,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +31: [2023-04-29 09:45:12,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +41: [2023-04-29 09:45:12,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +31: [2023-04-29 09:45:12,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +26: [2023-04-29 09:45:12,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +11: [2023-04-29 09:45:12,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +36: [2023-04-29 09:45:12,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +35: [2023-04-29 09:45:12,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +11: [2023-04-29 09:45:12,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +45: [2023-04-29 09:45:12,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:12,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +42: [2023-04-29 09:45:12,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +11: [2023-04-29 09:45:12,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +11: [2023-04-29 09:45:12,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +12: [2023-04-29 09:45:12,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 2: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +22: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +22: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +45: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +16: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +45: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +15: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +35: [2023-04-29 09:45:12,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:12,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +22: [2023-04-29 09:45:12,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +16: [2023-04-29 09:45:12,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +48: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +48: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +17: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +15: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +48: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +48: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +15: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +46: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:12,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +15: [2023-04-29 09:45:12,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +12: [2023-04-29 09:45:12,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:12,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +16: [2023-04-29 09:45:12,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +51: [2023-04-29 09:45:12,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:12,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:12,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:12,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +42: [2023-04-29 09:45:12,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +22: [2023-04-29 09:45:12,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:12,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +51: [2023-04-29 09:45:12,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +32: [2023-04-29 09:45:12,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:12,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +51: [2023-04-29 09:45:12,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:12,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +43: [2023-04-29 09:45:12,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +51: [2023-04-29 09:45:12,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +47: [2023-04-29 09:45:12,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +47: [2023-04-29 09:45:12,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +45: [2023-04-29 09:45:12,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:12,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +36: [2023-04-29 09:45:12,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +25: [2023-04-29 09:45:12,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +36: [2023-04-29 09:45:12,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +31: [2023-04-29 09:45:12,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +36: [2023-04-29 09:45:12,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +25: [2023-04-29 09:45:12,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +25: [2023-04-29 09:45:12,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +43: [2023-04-29 09:45:12,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +16: [2023-04-29 09:45:12,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:12,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +25: [2023-04-29 09:45:12,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +25: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +16: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +26: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +43: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +25: [2023-04-29 09:45:12,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +10: [2023-04-29 09:45:12,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +10: [2023-04-29 09:45:12,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +25: [2023-04-29 09:45:12,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +21: [2023-04-29 09:45:12,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +52: [2023-04-29 09:45:12,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +52: [2023-04-29 09:45:12,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_00-model_states.pt. +57: [2023-04-29 09:45:12,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +57: [2023-04-29 09:45:12,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +57: [2023-04-29 09:45:12,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +57: [2023-04-29 09:45:12,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +22: [2023-04-29 09:45:12,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:12,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:12,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:12,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:12,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +10: [2023-04-29 09:45:12,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:12,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +22: [2023-04-29 09:45:12,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:12,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +48: [2023-04-29 09:45:12,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +48: [2023-04-29 09:45:12,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +52: [2023-04-29 09:45:12,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +48: [2023-04-29 09:45:12,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +43: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:12,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +26: [2023-04-29 09:45:12,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +47: [2023-04-29 09:45:12,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +24: [2023-04-29 09:45:12,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +59: [2023-04-29 09:45:12,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +26: [2023-04-29 09:45:12,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +59: [2023-04-29 09:45:12,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +47: [2023-04-29 09:45:12,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +50: [2023-04-29 09:45:12,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +50: [2023-04-29 09:45:12,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +50: [2023-04-29 09:45:12,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +17: [2023-04-29 09:45:12,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +17: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +17: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 7: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +24: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +24: [2023-04-29 09:45:12,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +43: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 9: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +59: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 9: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +59: [2023-04-29 09:45:12,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +26: [2023-04-29 09:45:12,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +59: [2023-04-29 09:45:12,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +43: [2023-04-29 09:45:12,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +17: [2023-04-29 09:45:12,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +21: [2023-04-29 09:45:12,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +52: [2023-04-29 09:45:12,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +59: [2023-04-29 09:45:12,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +17: [2023-04-29 09:45:12,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +10: [2023-04-29 09:45:12,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +17: [2023-04-29 09:45:12,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:12,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +17: [2023-04-29 09:45:12,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +57: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +52: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +11: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +52: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 5: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +40: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +40: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +40: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +59: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +40: [2023-04-29 09:45:12,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +57: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. +57: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_15-model_01-model_states.pt. + 2: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 2: [2023-04-29 09:45:12,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +44: [2023-04-29 09:45:12,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +44: [2023-04-29 09:45:12,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +44: [2023-04-29 09:45:12,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +11: [2023-04-29 09:45:12,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +21: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +44: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 5: [2023-04-29 09:45:12,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +50: [2023-04-29 09:45:12,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +56: [2023-04-29 09:45:12,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +56: [2023-04-29 09:45:12,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +50: [2023-04-29 09:45:12,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +50: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:12,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +15: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +17: [2023-04-29 09:45:12,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +13: [2023-04-29 09:45:12,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +15: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +23: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +15: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +26: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +51: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +13: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +23: [2023-04-29 09:45:12,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 1: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +23: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +23: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:12,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +17: [2023-04-29 09:45:12,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +50: [2023-04-29 09:45:12,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +23: [2023-04-29 09:45:12,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 1: [2023-04-29 09:45:12,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:12,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +23: [2023-04-29 09:45:12,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +31: [2023-04-29 09:45:12,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +50: [2023-04-29 09:45:12,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +20: [2023-04-29 09:45:12,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:12,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:12,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +17: [2023-04-29 09:45:12,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +26: [2023-04-29 09:45:12,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:12,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:12,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +21: [2023-04-29 09:45:12,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:12,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:12,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:12,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +59: [2023-04-29 09:45:12,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +37: [2023-04-29 09:45:12,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +24: [2023-04-29 09:45:12,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:12,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +51: [2023-04-29 09:45:12,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +59: [2023-04-29 09:45:12,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:12,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +51: [2023-04-29 09:45:12,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +25: [2023-04-29 09:45:12,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +31: [2023-04-29 09:45:12,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +50: [2023-04-29 09:45:12,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +17: [2023-04-29 09:45:12,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +40: [2023-04-29 09:45:12,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +20: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +59: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +37: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +20: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +37: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +11: [2023-04-29 09:45:12,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +37: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +44: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +51: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +31: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +44: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +11: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +51: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +56: [2023-04-29 09:45:12,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +11: [2023-04-29 09:45:12,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +11: [2023-04-29 09:45:12,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:12,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 5: [2023-04-29 09:45:12,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +44: [2023-04-29 09:45:12,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +21: [2023-04-29 09:45:12,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +21: [2023-04-29 09:45:12,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +25: [2023-04-29 09:45:12,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +15: [2023-04-29 09:45:12,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +56: [2023-04-29 09:45:12,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:12,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:12,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:12,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +48: [2023-04-29 09:45:12,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +15: [2023-04-29 09:45:12,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +15: [2023-04-29 09:45:12,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +49: [2023-04-29 09:45:12,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +49: [2023-04-29 09:45:12,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +51: [2023-04-29 09:45:12,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +49: [2023-04-29 09:45:12,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +31: [2023-04-29 09:45:12,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +31: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +24: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 3: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +24: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +24: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 3: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +49: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +31: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +15: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +25: [2023-04-29 09:45:12,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +49: [2023-04-29 09:45:12,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +48: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:12,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +33: [2023-04-29 09:45:12,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +40: [2023-04-29 09:45:12,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +33: [2023-04-29 09:45:12,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +40: [2023-04-29 09:45:12,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +25: [2023-04-29 09:45:12,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +33: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 3: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +48: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +33: [2023-04-29 09:45:12,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +33: [2023-04-29 09:45:12,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +33: [2023-04-29 09:45:12,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +44: [2023-04-29 09:45:12,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +17: [2023-04-29 09:45:12,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +40: [2023-04-29 09:45:12,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:12,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +48: [2023-04-29 09:45:12,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +27: [2023-04-29 09:45:12,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +48: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +17: [2023-04-29 09:45:12,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:12,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +35: [2023-04-29 09:45:12,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +35: [2023-04-29 09:45:12,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +35: [2023-04-29 09:45:12,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +35: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +27: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +27: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +35: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +30: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +35: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +35: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +25: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +24: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:12,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +25: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +27: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +18: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +35: [2023-04-29 09:45:12,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +17: [2023-04-29 09:45:12,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +35: [2023-04-29 09:45:12,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +35: [2023-04-29 09:45:12,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +27: [2023-04-29 09:45:12,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +27: [2023-04-29 09:45:12,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +27: [2023-04-29 09:45:12,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +27: [2023-04-29 09:45:12,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +24: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +18: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +18: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +24: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:12,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +30: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +53: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +30: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:12,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:12,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +30: [2023-04-29 09:45:12,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +17: [2023-04-29 09:45:12,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:12,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +13: [2023-04-29 09:45:12,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +30: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 5: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +35: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +35: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +50: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +31: [2023-04-29 09:45:12,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +48: [2023-04-29 09:45:12,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:12,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +18: [2023-04-29 09:45:12,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +18: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +48: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +18: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 0: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +53: [2023-04-29 09:45:12,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:12,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:12,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +53: [2023-04-29 09:45:12,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:12,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +13: [2023-04-29 09:45:12,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +20: [2023-04-29 09:45:12,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +17: [2023-04-29 09:45:12,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +40: [2023-04-29 09:45:12,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +31: [2023-04-29 09:45:12,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +20: [2023-04-29 09:45:12,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:12,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +37: [2023-04-29 09:45:12,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:12,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:12,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:12,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:12,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +31: [2023-04-29 09:45:12,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +23: [2023-04-29 09:45:12,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:12,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:12,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +40: [2023-04-29 09:45:12,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +40: [2023-04-29 09:45:12,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +17: [2023-04-29 09:45:12,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +17: [2023-04-29 09:45:12,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +44: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +31: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +50: [2023-04-29 09:45:12,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +50: [2023-04-29 09:45:12,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:12,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 5: [2023-04-29 09:45:12,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +33: [2023-04-29 09:45:12,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:12,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +40: [2023-04-29 09:45:12,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +40: [2023-04-29 09:45:12,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +44: [2023-04-29 09:45:12,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +44: [2023-04-29 09:45:12,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +50: [2023-04-29 09:45:12,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:12,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +20: [2023-04-29 09:45:12,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +33: [2023-04-29 09:45:12,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +13: [2023-04-29 09:45:12,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:12,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +37: [2023-04-29 09:45:12,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:12,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +31: [2023-04-29 09:45:12,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +49: [2023-04-29 09:45:12,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 0: [2023-04-29 09:45:12,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +41: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 5: [2023-04-29 09:45:12,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +41: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +41: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 0: [2023-04-29 09:45:12,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +33: [2023-04-29 09:45:12,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +31: [2023-04-29 09:45:12,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:12,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 8: [2023-04-29 09:45:12,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +50: [2023-04-29 09:45:12,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +50: [2023-04-29 09:45:12,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +40: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:12,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +37: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +40: [2023-04-29 09:45:12,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +31: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 3: [2023-04-29 09:45:12,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +20: [2023-04-29 09:45:12,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +19: [2023-04-29 09:45:12,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +44: [2023-04-29 09:45:12,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:12,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +19: [2023-04-29 09:45:12,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +34: [2023-04-29 09:45:12,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:12,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:12,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +49: [2023-04-29 09:45:12,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:12,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +35: [2023-04-29 09:45:12,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +20: [2023-04-29 09:45:12,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:12,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:12,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:12,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +39: [2023-04-29 09:45:12,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:12,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:12,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +39: [2023-04-29 09:45:12,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:12,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:12,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:12,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +50: [2023-04-29 09:45:12,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +27: [2023-04-29 09:45:12,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +33: [2023-04-29 09:45:12,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 9: [2023-04-29 09:45:12,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 9: [2023-04-29 09:45:12,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +35: [2023-04-29 09:45:12,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +19: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +35: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +33: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +19: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. + 8: [2023-04-29 09:45:12,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +39: [2023-04-29 09:45:12,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:12,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +50: [2023-04-29 09:45:12,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +39: [2023-04-29 09:45:12,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:12,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:12,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +19: [2023-04-29 09:45:12,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:12,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +27: [2023-04-29 09:45:12,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +24: [2023-04-29 09:45:12,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +19: [2023-04-29 09:45:12,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... + 9: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +63: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +54: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +24: [2023-04-29 09:45:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +50: [2023-04-29 09:45:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:12,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +53: [2023-04-29 09:45:12,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +30: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +30: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:12,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:12,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +63: [2023-04-29 09:45:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +53: [2023-04-29 09:45:12,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:12,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +18: [2023-04-29 09:45:12,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +18: [2023-04-29 09:45:12,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +19: [2023-04-29 09:45:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt... +63: [2023-04-29 09:45:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +30: [2023-04-29 09:45:12,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +28: [2023-04-29 09:45:12,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +63: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +19: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +53: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +19: [2023-04-29 09:45:12,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt... +28: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +57: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +57: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +28: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +57: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +28: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +57: [2023-04-29 09:45:12,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +33: [2023-04-29 09:45:12,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:12,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +24: [2023-04-29 09:45:12,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +33: [2023-04-29 09:45:12,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 1: [2023-04-29 09:45:12,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 1: [2023-04-29 09:45:12,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 1: [2023-04-29 09:45:12,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +40: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +35: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +49: [2023-04-29 09:45:12,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:12,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +56: [2023-04-29 09:45:12,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +62: [2023-04-29 09:45:12,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +32: [2023-04-29 09:45:12,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:12,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:12,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:12,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:12,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:12,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:12,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +62: [2023-04-29 09:45:12,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +56: [2023-04-29 09:45:12,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:12,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +30: [2023-04-29 09:45:12,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:12,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +62: [2023-04-29 09:45:12,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +56: [2023-04-29 09:45:12,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +56: [2023-04-29 09:45:12,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +56: [2023-04-29 09:45:12,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +56: [2023-04-29 09:45:12,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +32: [2023-04-29 09:45:12,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:12,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +32: [2023-04-29 09:45:12,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +56: [2023-04-29 09:45:12,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +32: [2023-04-29 09:45:12,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:12,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:12,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +42: [2023-04-29 09:45:12,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:12,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +35: [2023-04-29 09:45:12,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:12,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +56: [2023-04-29 09:45:12,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +35: [2023-04-29 09:45:12,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:12,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +42: [2023-04-29 09:45:12,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +27: [2023-04-29 09:45:12,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:12,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:12,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:12,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +27: [2023-04-29 09:45:12,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +40: [2023-04-29 09:45:12,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:12,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +40: [2023-04-29 09:45:12,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +30: [2023-04-29 09:45:12,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:12,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +30: [2023-04-29 09:45:12,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +53: [2023-04-29 09:45:12,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +30: [2023-04-29 09:45:12,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:12,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +53: [2023-04-29 09:45:12,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:12,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +53: [2023-04-29 09:45:12,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +35: [2023-04-29 09:45:12,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:12,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:12,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +34: [2023-04-29 09:45:12,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +28: [2023-04-29 09:45:12,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +28: [2023-04-29 09:45:12,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +28: [2023-04-29 09:45:12,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +62: [2023-04-29 09:45:12,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:12,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:12,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:12,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:12,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:12,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +62: [2023-04-29 09:45:12,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:12,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:12,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:12,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +51: [2023-04-29 09:45:12,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +29: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +29: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +58: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +58: [2023-04-29 09:45:12,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +58: [2023-04-29 09:45:12,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:12,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +39: [2023-04-29 09:45:12,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:12,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +29: [2023-04-29 09:45:12,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:12,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:12,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +14: [2023-04-29 09:45:12,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +41: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:12,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +51: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:12,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:12,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:12,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:12,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:12,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +55: [2023-04-29 09:45:12,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +55: [2023-04-29 09:45:12,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +55: [2023-04-29 09:45:12,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +55: [2023-04-29 09:45:12,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +41: [2023-04-29 09:45:12,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:12,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +61: [2023-04-29 09:45:12,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +61: [2023-04-29 09:45:12,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +14: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +14: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +38: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:12,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +34: [2023-04-29 09:45:12,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:12,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:12,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:12,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +60: [2023-04-29 09:45:12,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:12,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:12,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +60: [2023-04-29 09:45:12,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +38: [2023-04-29 09:45:12,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +38: [2023-04-29 09:45:12,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:12,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:12,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:12,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:12,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +57: [2023-04-29 09:45:12,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +19: [2023-04-29 09:45:12,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +60: [2023-04-29 09:45:12,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +52: [2023-04-29 09:45:12,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +52: [2023-04-29 09:45:12,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +57: [2023-04-29 09:45:12,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +34: [2023-04-29 09:45:12,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:12,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +63: [2023-04-29 09:45:12,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:12,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:12,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +60: [2023-04-29 09:45:12,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +52: [2023-04-29 09:45:12,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +34: [2023-04-29 09:45:12,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:12,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +60: [2023-04-29 09:45:12,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +60: [2023-04-29 09:45:12,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +60: [2023-04-29 09:45:12,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +19: [2023-04-29 09:45:12,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +57: [2023-04-29 09:45:12,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +41: [2023-04-29 09:45:12,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:12,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +56: [2023-04-29 09:45:12,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +41: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +16: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +19: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_00-model_states.pt. +28: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +16: [2023-04-29 09:45:12,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:12,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:12,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +56: [2023-04-29 09:45:12,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +28: [2023-04-29 09:45:12,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +63: [2023-04-29 09:45:12,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:12,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +39: [2023-04-29 09:45:12,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +12: [2023-04-29 09:45:12,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +12: [2023-04-29 09:45:12,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +32: [2023-04-29 09:45:12,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:12,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +12: [2023-04-29 09:45:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +41: [2023-04-29 09:45:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +16: [2023-04-29 09:45:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +55: [2023-04-29 09:45:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +56: [2023-04-29 09:45:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +16: [2023-04-29 09:45:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +61: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +61: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +12: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +42: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +55: [2023-04-29 09:45:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +55: [2023-04-29 09:45:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +16: [2023-04-29 09:45:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +39: [2023-04-29 09:45:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +61: [2023-04-29 09:45:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +54: [2023-04-29 09:45:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +54: [2023-04-29 09:45:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +46: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +46: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +45: [2023-04-29 09:45:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +63: [2023-04-29 09:45:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +46: [2023-04-29 09:45:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +32: [2023-04-29 09:45:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:12,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:12,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:12,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:12,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:12,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:12,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:12,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +45: [2023-04-29 09:45:12,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +45: [2023-04-29 09:45:12,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:12,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +52: [2023-04-29 09:45:12,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +45: [2023-04-29 09:45:12,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:12,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +19: [2023-04-29 09:45:12,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +28: [2023-04-29 09:45:12,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +52: [2023-04-29 09:45:12,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:12,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:12,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +29: [2023-04-29 09:45:12,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +36: [2023-04-29 09:45:12,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:12,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +36: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +36: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +36: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +60: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +57: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:12,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +62: [2023-04-29 09:45:12,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +54: [2023-04-29 09:45:12,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +43: [2023-04-29 09:45:12,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +43: [2023-04-29 09:45:12,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +43: [2023-04-29 09:45:12,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +54: [2023-04-29 09:45:12,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:12,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:12,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:12,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +29: [2023-04-29 09:45:12,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +58: [2023-04-29 09:45:12,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +43: [2023-04-29 09:45:12,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:12,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:12,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +19: [2023-04-29 09:45:12,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:12,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +62: [2023-04-29 09:45:12,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:12,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:12,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:12,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:12,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +54: [2023-04-29 09:45:12,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +56: [2023-04-29 09:45:12,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 1: [2023-04-29 09:45:12,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +32: [2023-04-29 09:45:12,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +28: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +51: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:12,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:12,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +54: [2023-04-29 09:45:12,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:12,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:12,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:12,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:12,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:12,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:12,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +59: [2023-04-29 09:45:12,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:12,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +56: [2023-04-29 09:45:12,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:12,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 1: [2023-04-29 09:45:12,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +55: [2023-04-29 09:45:12,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +55: [2023-04-29 09:45:12,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +29: [2023-04-29 09:45:12,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +55: [2023-04-29 09:45:12,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:12,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:12,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +58: [2023-04-29 09:45:12,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:12,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +29: [2023-04-29 09:45:12,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:12,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:12,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:12,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:12,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:12,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +29: [2023-04-29 09:45:12,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:12,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:12,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:12,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:12,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +38: [2023-04-29 09:45:12,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:12,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:12,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +62: [2023-04-29 09:45:12,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 6: [2023-04-29 09:45:12,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 6: [2023-04-29 09:45:12,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +61: [2023-04-29 09:45:12,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:12,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +61: [2023-04-29 09:45:12,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:12,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +62: [2023-04-29 09:45:12,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:12,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +62: [2023-04-29 09:45:12,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +12: [2023-04-29 09:45:12,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +16: [2023-04-29 09:45:12,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:12,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +51: [2023-04-29 09:45:12,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:12,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:12,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +61: [2023-04-29 09:45:12,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +12: [2023-04-29 09:45:12,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +38: [2023-04-29 09:45:12,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:12,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +61: [2023-04-29 09:45:12,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +39: [2023-04-29 09:45:12,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +39: [2023-04-29 09:45:12,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +58: [2023-04-29 09:45:12,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +16: [2023-04-29 09:45:12,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +39: [2023-04-29 09:45:12,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +28: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +39: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +51: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +47: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +62: [2023-04-29 09:45:12,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +34: [2023-04-29 09:45:12,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +34: [2023-04-29 09:45:12,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +34: [2023-04-29 09:45:12,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +34: [2023-04-29 09:45:12,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +12: [2023-04-29 09:45:12,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:12,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:12,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:12,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:12,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +47: [2023-04-29 09:45:12,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +47: [2023-04-29 09:45:12,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +46: [2023-04-29 09:45:12,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:12,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +20: [2023-04-29 09:45:12,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +20: [2023-04-29 09:45:12,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +20: [2023-04-29 09:45:12,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +20: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +52: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +52: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +22: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +52: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +22: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +60: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +14: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +45: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +55: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +22: [2023-04-29 09:45:12,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 4: [2023-04-29 09:45:12,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +47: [2023-04-29 09:45:12,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 4: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +45: [2023-04-29 09:45:12,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +45: [2023-04-29 09:45:12,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +45: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +45: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +22: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +61: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:12,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +14: [2023-04-29 09:45:12,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +47: [2023-04-29 09:45:12,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +38: [2023-04-29 09:45:12,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:12,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +47: [2023-04-29 09:45:12,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +29: [2023-04-29 09:45:12,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:12,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +52: [2023-04-29 09:45:12,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +26: [2023-04-29 09:45:12,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:12,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +47: [2023-04-29 09:45:12,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +52: [2023-04-29 09:45:12,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +26: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +60: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +26: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +52: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +28: [2023-04-29 09:45:12,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:12,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +52: [2023-04-29 09:45:12,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +10: [2023-04-29 09:45:12,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:12,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +61: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +61: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +55: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +55: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +16: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +55: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +55: [2023-04-29 09:45:12,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +61: [2023-04-29 09:45:12,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 6: [2023-04-29 09:45:12,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:12,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +26: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +26: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +12: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +38: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +10: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +61: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +26: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +38: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:12,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:12,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +16: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 6: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +38: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +38: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +10: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +52: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +46: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:12,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +28: [2023-04-29 09:45:12,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +10: [2023-04-29 09:45:12,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:12,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +21: [2023-04-29 09:45:12,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:12,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:12,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +12: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +12: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +52: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +12: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +21: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +54: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 6: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +43: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:12,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:12,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +58: [2023-04-29 09:45:12,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +58: [2023-04-29 09:45:12,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +58: [2023-04-29 09:45:12,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +60: [2023-04-29 09:45:12,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +60: [2023-04-29 09:45:12,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +54: [2023-04-29 09:45:12,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +62: [2023-04-29 09:45:12,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +62: [2023-04-29 09:45:12,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:12,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +12: [2023-04-29 09:45:12,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:12,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +21: [2023-04-29 09:45:12,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:12,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +46: [2023-04-29 09:45:12,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. + 7: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +45: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +21: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +39: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:12,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +21: [2023-04-29 09:45:12,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:12,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +43: [2023-04-29 09:45:12,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +43: [2023-04-29 09:45:12,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:12,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +21: [2023-04-29 09:45:12,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +12: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +49: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +46: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +34: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +47: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +34: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:12,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +16: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +50: [2023-04-29 09:45:12,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:12,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:12,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:12,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:12,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:12,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:12,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +28: [2023-04-29 09:45:12,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:12,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +46: [2023-04-29 09:45:12,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +20: [2023-04-29 09:45:12,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +47: [2023-04-29 09:45:12,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +59: [2023-04-29 09:45:12,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +60: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +52: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +28: [2023-04-29 09:45:12,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:12,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:12,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:12,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:12,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:12,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:12,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:12,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:12,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:12,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +45: [2023-04-29 09:45:12,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:12,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:12,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:12,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:12,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:12,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:12,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +40: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 4: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +40: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +60: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +40: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:12,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:12,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +43: [2023-04-29 09:45:12,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:12,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +52: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +59: [2023-04-29 09:45:12,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:12,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +60: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +53: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +55: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +55: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +53: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +45: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +53: [2023-04-29 09:45:12,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +20: [2023-04-29 09:45:12,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:12,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +11: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +28: [2023-04-29 09:45:12,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +14: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +14: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +20: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +14: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +14: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +40: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +38: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 7: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +40: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +38: [2023-04-29 09:45:12,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:12,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +38: [2023-04-29 09:45:12,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:12,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:12,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:13,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:13,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +43: [2023-04-29 09:45:13,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +45: [2023-04-29 09:45:13,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +46: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +63: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +63: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +63: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +12: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +63: [2023-04-29 09:45:13,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_37-model_01-model_states.pt. +11: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +11: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +11: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +11: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +38: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +58: [2023-04-29 09:45:13,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:13,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +12: [2023-04-29 09:45:13,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +37: [2023-04-29 09:45:13,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +36: [2023-04-29 09:45:13,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +17: [2023-04-29 09:45:13,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +37: [2023-04-29 09:45:13,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +36: [2023-04-29 09:45:13,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +17: [2023-04-29 09:45:13,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +37: [2023-04-29 09:45:13,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +43: [2023-04-29 09:45:13,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:13,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +33: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +33: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +61: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +17: [2023-04-29 09:45:13,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +37: [2023-04-29 09:45:13,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:13,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +37: [2023-04-29 09:45:13,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +17: [2023-04-29 09:45:13,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +45: [2023-04-29 09:45:13,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +45: [2023-04-29 09:45:13,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:13,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +44: [2023-04-29 09:45:13,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:13,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +45: [2023-04-29 09:45:13,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +44: [2023-04-29 09:45:13,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:13,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +35: [2023-04-29 09:45:13,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:13,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:13,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +33: [2023-04-29 09:45:13,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:13,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +44: [2023-04-29 09:45:13,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +61: [2023-04-29 09:45:13,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +44: [2023-04-29 09:45:13,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:13,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +22: [2023-04-29 09:45:13,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +44: [2023-04-29 09:45:13,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +35: [2023-04-29 09:45:13,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +10: [2023-04-29 09:45:13,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +35: [2023-04-29 09:45:13,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:13,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +53: [2023-04-29 09:45:13,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:13,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +14: [2023-04-29 09:45:13,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +35: [2023-04-29 09:45:13,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:13,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +35: [2023-04-29 09:45:13,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +61: [2023-04-29 09:45:13,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +59: [2023-04-29 09:45:13,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:13,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +35: [2023-04-29 09:45:13,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +22: [2023-04-29 09:45:13,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:13,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:13,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +53: [2023-04-29 09:45:13,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:13,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +22: [2023-04-29 09:45:13,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +52: [2023-04-29 09:45:13,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:13,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +47: [2023-04-29 09:45:13,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +63: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +63: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... +41: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +41: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +41: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +26: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +41: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +32: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +32: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +32: [2023-04-29 09:45:13,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +32: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +55: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +48: [2023-04-29 09:45:13,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +48: [2023-04-29 09:45:13,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +48: [2023-04-29 09:45:13,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +48: [2023-04-29 09:45:13,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +61: [2023-04-29 09:45:13,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:13,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +14: [2023-04-29 09:45:13,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +14: [2023-04-29 09:45:13,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +26: [2023-04-29 09:45:13,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +25: [2023-04-29 09:45:13,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +25: [2023-04-29 09:45:13,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +52: [2023-04-29 09:45:13,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +25: [2023-04-29 09:45:13,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +48: [2023-04-29 09:45:13,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +25: [2023-04-29 09:45:13,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +48: [2023-04-29 09:45:13,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:13,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:13,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +25: [2023-04-29 09:45:13,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +47: [2023-04-29 09:45:13,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:13,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +25: [2023-04-29 09:45:13,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:13,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +10: [2023-04-29 09:45:13,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +50: [2023-04-29 09:45:13,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +25: [2023-04-29 09:45:13,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:13,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:13,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +39: [2023-04-29 09:45:13,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +25: [2023-04-29 09:45:13,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +21: [2023-04-29 09:45:13,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:13,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +21: [2023-04-29 09:45:13,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +34: [2023-04-29 09:45:13,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:13,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:13,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:13,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +39: [2023-04-29 09:45:13,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:13,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:13,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:13,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +22: [2023-04-29 09:45:13,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:13,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +22: [2023-04-29 09:45:13,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +39: [2023-04-29 09:45:13,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +20: [2023-04-29 09:45:13,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +34: [2023-04-29 09:45:13,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +34: [2023-04-29 09:45:13,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:13,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +39: [2023-04-29 09:45:13,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +45: [2023-04-29 09:45:13,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +26: [2023-04-29 09:45:13,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +38: [2023-04-29 09:45:13,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +34: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +47: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +26: [2023-04-29 09:45:13,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +24: [2023-04-29 09:45:13,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +39: [2023-04-29 09:45:13,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +47: [2023-04-29 09:45:13,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +32: [2023-04-29 09:45:13,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:13,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +26: [2023-04-29 09:45:13,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +32: [2023-04-29 09:45:13,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +10: [2023-04-29 09:45:13,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +32: [2023-04-29 09:45:13,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:13,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +38: [2023-04-29 09:45:13,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:13,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +22: [2023-04-29 09:45:13,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +47: [2023-04-29 09:45:13,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +47: [2023-04-29 09:45:13,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:13,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:13,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:13,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +41: [2023-04-29 09:45:13,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:13,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:13,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +49: [2023-04-29 09:45:13,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +15: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +40: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +40: [2023-04-29 09:45:13,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +11: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +58: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +38: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +11: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +38: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:13,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +15: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +58: [2023-04-29 09:45:13,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +45: [2023-04-29 09:45:13,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +26: [2023-04-29 09:45:13,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +58: [2023-04-29 09:45:13,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +41: [2023-04-29 09:45:13,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:13,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +11: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +47: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +15: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +12: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +17: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +47: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +47: [2023-04-29 09:45:13,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +20: [2023-04-29 09:45:13,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +45: [2023-04-29 09:45:13,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +58: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +49: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +38: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:13,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:13,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +47: [2023-04-29 09:45:13,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +33: [2023-04-29 09:45:13,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +15: [2023-04-29 09:45:13,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +58: [2023-04-29 09:45:13,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +17: [2023-04-29 09:45:13,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +47: [2023-04-29 09:45:13,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +44: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +21: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:13,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:13,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +47: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +20: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +38: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +33: [2023-04-29 09:45:13,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +46: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +21: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +46: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +24: [2023-04-29 09:45:13,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +45: [2023-04-29 09:45:13,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:13,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +12: [2023-04-29 09:45:13,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +37: [2023-04-29 09:45:13,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +45: [2023-04-29 09:45:13,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +20: [2023-04-29 09:45:13,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +20: [2023-04-29 09:45:13,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +33: [2023-04-29 09:45:13,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +33: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +49: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +20: [2023-04-29 09:45:13,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +53: [2023-04-29 09:45:13,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:13,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +46: [2023-04-29 09:45:13,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +14: [2023-04-29 09:45:13,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +46: [2023-04-29 09:45:13,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +35: [2023-04-29 09:45:13,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:13,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +49: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +45: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +47: [2023-04-29 09:45:13,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:13,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:13,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +35: [2023-04-29 09:45:13,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:13,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +63: [2023-04-29 09:45:13,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +11: [2023-04-29 09:45:13,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:13,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +40: [2023-04-29 09:45:13,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:13,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +11: [2023-04-29 09:45:13,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +40: [2023-04-29 09:45:13,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +49: [2023-04-29 09:45:13,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +17: [2023-04-29 09:45:13,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:13,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +11: [2023-04-29 09:45:13,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +53: [2023-04-29 09:45:13,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:13,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +33: [2023-04-29 09:45:13,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:13,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +48: [2023-04-29 09:45:13,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +42: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +42: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +13: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +25: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +44: [2023-04-29 09:45:13,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +24: [2023-04-29 09:45:13,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +24: [2023-04-29 09:45:13,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +53: [2023-04-29 09:45:13,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:13,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +44: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +24: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +63: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +63: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +17: [2023-04-29 09:45:13,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +24: [2023-04-29 09:45:13,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +63: [2023-04-29 09:45:13,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_00-model_states.pt. +37: [2023-04-29 09:45:13,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:13,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +37: [2023-04-29 09:45:13,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +24: [2023-04-29 09:45:13,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:13,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +14: [2023-04-29 09:45:13,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +48: [2023-04-29 09:45:13,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +48: [2023-04-29 09:45:13,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +53: [2023-04-29 09:45:13,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +33: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +53: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +63: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +56: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +56: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +56: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +56: [2023-04-29 09:45:13,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:13,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +33: [2023-04-29 09:45:13,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +24: [2023-04-29 09:45:13,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +48: [2023-04-29 09:45:13,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +24: [2023-04-29 09:45:13,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +57: [2023-04-29 09:45:13,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +25: [2023-04-29 09:45:13,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +63: [2023-04-29 09:45:13,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt... +32: [2023-04-29 09:45:13,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +56: [2023-04-29 09:45:13,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:13,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +56: [2023-04-29 09:45:13,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +56: [2023-04-29 09:45:13,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:13,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +33: [2023-04-29 09:45:13,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +53: [2023-04-29 09:45:13,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +53: [2023-04-29 09:45:13,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +56: [2023-04-29 09:45:13,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:13,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +14: [2023-04-29 09:45:13,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +25: [2023-04-29 09:45:13,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +57: [2023-04-29 09:45:13,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:13,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:13,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:13,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +35: [2023-04-29 09:45:13,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +14: [2023-04-29 09:45:13,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +57: [2023-04-29 09:45:13,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +44: [2023-04-29 09:45:13,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +11: [2023-04-29 09:45:13,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +32: [2023-04-29 09:45:13,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:13,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +11: [2023-04-29 09:45:13,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +43: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +43: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +43: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +43: [2023-04-29 09:45:13,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +24: [2023-04-29 09:45:13,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +48: [2023-04-29 09:45:13,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:13,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:13,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:13,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:13,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:13,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +25: [2023-04-29 09:45:13,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +15: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +32: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +32: [2023-04-29 09:45:13,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +10: [2023-04-29 09:45:13,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +10: [2023-04-29 09:45:13,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +10: [2023-04-29 09:45:13,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +10: [2023-04-29 09:45:13,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +41: [2023-04-29 09:45:13,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:13,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:13,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:13,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:13,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:13,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +48: [2023-04-29 09:45:13,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +48: [2023-04-29 09:45:13,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +41: [2023-04-29 09:45:13,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:13,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +41: [2023-04-29 09:45:13,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +25: [2023-04-29 09:45:13,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +63: [2023-04-29 09:45:13,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +63: [2023-04-29 09:45:13,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +42: [2023-04-29 09:45:13,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +15: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +15: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +63: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +31: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +31: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +63: [2023-04-29 09:45:13,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +24: [2023-04-29 09:45:13,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +11: [2023-04-29 09:45:13,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +41: [2023-04-29 09:45:13,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +31: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +31: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +41: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +42: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +34: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +36: [2023-04-29 09:45:13,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +22: [2023-04-29 09:45:13,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +22: [2023-04-29 09:45:13,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +22: [2023-04-29 09:45:13,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +22: [2023-04-29 09:45:13,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +42: [2023-04-29 09:45:13,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +11: [2023-04-29 09:45:13,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +25: [2023-04-29 09:45:13,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +43: [2023-04-29 09:45:13,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +16: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +16: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +16: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +42: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:13,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +41: [2023-04-29 09:45:13,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +43: [2023-04-29 09:45:13,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:13,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:13,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +31: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +59: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +34: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +59: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +59: [2023-04-29 09:45:13,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +43: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +34: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:13,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +62: [2023-04-29 09:45:13,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +60: [2023-04-29 09:45:13,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +60: [2023-04-29 09:45:13,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +15: [2023-04-29 09:45:13,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +15: [2023-04-29 09:45:13,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +60: [2023-04-29 09:45:13,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +60: [2023-04-29 09:45:13,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +62: [2023-04-29 09:45:13,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:13,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +62: [2023-04-29 09:45:13,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +39: [2023-04-29 09:45:13,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +39: [2023-04-29 09:45:13,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +10: [2023-04-29 09:45:13,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:13,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +60: [2023-04-29 09:45:13,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:13,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +20: [2023-04-29 09:45:13,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +39: [2023-04-29 09:45:13,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +39: [2023-04-29 09:45:13,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:13,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:13,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +60: [2023-04-29 09:45:13,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:13,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:13,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +39: [2023-04-29 09:45:13,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:13,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +23: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +23: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +10: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +13: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:13,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +15: [2023-04-29 09:45:13,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +22: [2023-04-29 09:45:13,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +22: [2023-04-29 09:45:13,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +22: [2023-04-29 09:45:13,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +22: [2023-04-29 09:45:13,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +16: [2023-04-29 09:45:13,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +20: [2023-04-29 09:45:13,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:13,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +51: [2023-04-29 09:45:13,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +57: [2023-04-29 09:45:13,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:13,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +59: [2023-04-29 09:45:13,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +51: [2023-04-29 09:45:13,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +19: [2023-04-29 09:45:13,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +51: [2023-04-29 09:45:13,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +16: [2023-04-29 09:45:13,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +41: [2023-04-29 09:45:13,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +16: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +33: [2023-04-29 09:45:13,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +33: [2023-04-29 09:45:13,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +33: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +18: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +33: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +36: [2023-04-29 09:45:13,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +53: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +18: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +56: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +19: [2023-04-29 09:45:13,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +57: [2023-04-29 09:45:13,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:13,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +56: [2023-04-29 09:45:13,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +13: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +36: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +13: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +59: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +13: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +41: [2023-04-29 09:45:13,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:13,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:13,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +41: [2023-04-29 09:45:13,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +18: [2023-04-29 09:45:13,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:13,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +59: [2023-04-29 09:45:13,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +18: [2023-04-29 09:45:13,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:13,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:13,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +24: [2023-04-29 09:45:13,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +24: [2023-04-29 09:45:13,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +42: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +28: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +28: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +28: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +28: [2023-04-29 09:45:13,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:13,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +20: [2023-04-29 09:45:13,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +54: [2023-04-29 09:45:13,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +20: [2023-04-29 09:45:13,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:13,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:13,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:13,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:13,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:13,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:13,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:13,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:13,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +53: [2023-04-29 09:45:13,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:13,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +13: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +57: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +56: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +42: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +56: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +27: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +54: [2023-04-29 09:45:13,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +54: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +63: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +27: [2023-04-29 09:45:13,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +33: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +63: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +63: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +11: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +54: [2023-04-29 09:45:13,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:13,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +11: [2023-04-29 09:45:13,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +51: [2023-04-29 09:45:13,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:13,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +57: [2023-04-29 09:45:13,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +30: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +51: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +24: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +33: [2023-04-29 09:45:13,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +34: [2023-04-29 09:45:13,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +43: [2023-04-29 09:45:13,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +27: [2023-04-29 09:45:13,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +56: [2023-04-29 09:45:13,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +51: [2023-04-29 09:45:13,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:13,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +53: [2023-04-29 09:45:13,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +53: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +27: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +27: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +62: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +30: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +42: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +11: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +11: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +53: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +34: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +13: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +24: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +13: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +33: [2023-04-29 09:45:13,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +33: [2023-04-29 09:45:13,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:13,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:13,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:13,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +11: [2023-04-29 09:45:13,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +11: [2023-04-29 09:45:13,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +24: [2023-04-29 09:45:13,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:13,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +24: [2023-04-29 09:45:13,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:13,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +42: [2023-04-29 09:45:13,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:13,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +60: [2023-04-29 09:45:13,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:13,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +38: [2023-04-29 09:45:13,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +38: [2023-04-29 09:45:13,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +10: [2023-04-29 09:45:13,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +46: [2023-04-29 09:45:13,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +43: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +10: [2023-04-29 09:45:13,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +55: [2023-04-29 09:45:13,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:13,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:13,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:13,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +63: [2023-04-29 09:45:13,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:13,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +29: [2023-04-29 09:45:13,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +29: [2023-04-29 09:45:13,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:13,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +63: [2023-04-29 09:45:13,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +10: [2023-04-29 09:45:13,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +10: [2023-04-29 09:45:13,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +43: [2023-04-29 09:45:13,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:13,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:13,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +62: [2023-04-29 09:45:13,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +43: [2023-04-29 09:45:13,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +39: [2023-04-29 09:45:13,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +39: [2023-04-29 09:45:13,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +22: [2023-04-29 09:45:13,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:13,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:13,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +59: [2023-04-29 09:45:13,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +22: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +42: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +61: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +22: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +21: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +10: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +43: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +31: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +19: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +19: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +12: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +12: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +22: [2023-04-29 09:45:13,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 3: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +18: [2023-04-29 09:45:13,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +12: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +59: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +12: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +16: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +16: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +61: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +61: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +36: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:13,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +41: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +22: [2023-04-29 09:45:13,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +60: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +49: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +49: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +49: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +22: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +18: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +49: [2023-04-29 09:45:13,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +59: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +10: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +48: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +48: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +48: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +48: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +23: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +12: [2023-04-29 09:45:13,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +22: [2023-04-29 09:45:13,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +10: [2023-04-29 09:45:13,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +34: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +37: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +37: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +37: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +59: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +36: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +34: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +37: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +41: [2023-04-29 09:45:13,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +26: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +26: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +26: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +42: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:13,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +42: [2023-04-29 09:45:13,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:13,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +62: [2023-04-29 09:45:13,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +26: [2023-04-29 09:45:13,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +59: [2023-04-29 09:45:13,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +16: [2023-04-29 09:45:13,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +16: [2023-04-29 09:45:13,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +36: [2023-04-29 09:45:13,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +36: [2023-04-29 09:45:13,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +28: [2023-04-29 09:45:13,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +60: [2023-04-29 09:45:13,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +28: [2023-04-29 09:45:13,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +28: [2023-04-29 09:45:13,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +28: [2023-04-29 09:45:13,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +10: [2023-04-29 09:45:13,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +39: [2023-04-29 09:45:13,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +39: [2023-04-29 09:45:13,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:13,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +23: [2023-04-29 09:45:13,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +36: [2023-04-29 09:45:13,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +59: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +60: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +36: [2023-04-29 09:45:13,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +27: [2023-04-29 09:45:13,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +27: [2023-04-29 09:45:13,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +39: [2023-04-29 09:45:13,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +36: [2023-04-29 09:45:13,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +18: [2023-04-29 09:45:13,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +36: [2023-04-29 09:45:13,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +23: [2023-04-29 09:45:13,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +36: [2023-04-29 09:45:13,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:13,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:13,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +35: [2023-04-29 09:45:13,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +35: [2023-04-29 09:45:13,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +39: [2023-04-29 09:45:13,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +62: [2023-04-29 09:45:13,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:13,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +19: [2023-04-29 09:45:13,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +23: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +30: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +33: [2023-04-29 09:45:13,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +35: [2023-04-29 09:45:13,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +35: [2023-04-29 09:45:13,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +21: [2023-04-29 09:45:13,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +36: [2023-04-29 09:45:13,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:13,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +13: [2023-04-29 09:45:13,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +16: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +16: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +16: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +36: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +18: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +37: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +41: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +41: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +27: [2023-04-29 09:45:13,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +27: [2023-04-29 09:45:13,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +15: [2023-04-29 09:45:13,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +48: [2023-04-29 09:45:13,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:13,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +29: [2023-04-29 09:45:13,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +41: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +31: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +50: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +50: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +37: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +48: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +13: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 3: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +48: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +13: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +29: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +41: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +52: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +16: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +15: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +26: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +45: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +45: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +45: [2023-04-29 09:45:13,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +30: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +45: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +19: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +50: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +19: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +13: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +16: [2023-04-29 09:45:13,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +50: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +37: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +26: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +51: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +16: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +18: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +48: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +23: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +49: [2023-04-29 09:45:13,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +14: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +23: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +31: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +37: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +23: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +33: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +13: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +51: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +51: [2023-04-29 09:45:13,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +26: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +52: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +26: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +45: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:13,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +45: [2023-04-29 09:45:13,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +52: [2023-04-29 09:45:13,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +13: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +51: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +45: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +23: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +13: [2023-04-29 09:45:13,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +14: [2023-04-29 09:45:13,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +33: [2023-04-29 09:45:13,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +23: [2023-04-29 09:45:13,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 5: [2023-04-29 09:45:13,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +14: [2023-04-29 09:45:13,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +13: [2023-04-29 09:45:13,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 5: [2023-04-29 09:45:13,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +33: [2023-04-29 09:45:13,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +38: [2023-04-29 09:45:13,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +23: [2023-04-29 09:45:13,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +51: [2023-04-29 09:45:13,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +51: [2023-04-29 09:45:13,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +46: [2023-04-29 09:45:13,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:13,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:13,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +33: [2023-04-29 09:45:13,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +10: [2023-04-29 09:45:13,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +10: [2023-04-29 09:45:13,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +27: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +51: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +10: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +33: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +35: [2023-04-29 09:45:13,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +38: [2023-04-29 09:45:13,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +35: [2023-04-29 09:45:13,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +28: [2023-04-29 09:45:13,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +30: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +23: [2023-04-29 09:45:13,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +38: [2023-04-29 09:45:13,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +56: [2023-04-29 09:45:13,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +54: [2023-04-29 09:45:13,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +56: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +56: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +10: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +35: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +43: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +28: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +57: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +28: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +55: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +10: [2023-04-29 09:45:13,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +15: [2023-04-29 09:45:13,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +29: [2023-04-29 09:45:13,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +10: [2023-04-29 09:45:13,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +35: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +40: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +40: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +28: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +28: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +19: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +40: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +29: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +50: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +29: [2023-04-29 09:45:13,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +31: [2023-04-29 09:45:13,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +55: [2023-04-29 09:45:13,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:13,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +19: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +17: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +17: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +12: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +25: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +25: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +50: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +50: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +27: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +27: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +27: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +27: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +17: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +29: [2023-04-29 09:45:13,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +27: [2023-04-29 09:45:13,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +17: [2023-04-29 09:45:13,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +61: [2023-04-29 09:45:13,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +43: [2023-04-29 09:45:13,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +18: [2023-04-29 09:45:13,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +18: [2023-04-29 09:45:13,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +50: [2023-04-29 09:45:13,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:13,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +12: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +54: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +38: [2023-04-29 09:45:13,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +43: [2023-04-29 09:45:13,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +44: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +44: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +44: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +44: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +18: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +22: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +32: [2023-04-29 09:45:13,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +26: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +26: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +26: [2023-04-29 09:45:13,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +15: [2023-04-29 09:45:13,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +15: [2023-04-29 09:45:13,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +32: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:13,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +12: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +26: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +12: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +22: [2023-04-29 09:45:13,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +46: [2023-04-29 09:45:13,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:13,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +22: [2023-04-29 09:45:13,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +58: [2023-04-29 09:45:13,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +26: [2023-04-29 09:45:13,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +21: [2023-04-29 09:45:13,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +26: [2023-04-29 09:45:13,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +43: [2023-04-29 09:45:13,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +56: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +29: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +26: [2023-04-29 09:45:13,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +26: [2023-04-29 09:45:13,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +22: [2023-04-29 09:45:13,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +59: [2023-04-29 09:45:13,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +59: [2023-04-29 09:45:13,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +29: [2023-04-29 09:45:13,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +38: [2023-04-29 09:45:13,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:13,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:13,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +21: [2023-04-29 09:45:13,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +56: [2023-04-29 09:45:13,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +56: [2023-04-29 09:45:13,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:13,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +21: [2023-04-29 09:45:13,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +21: [2023-04-29 09:45:13,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +21: [2023-04-29 09:45:13,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:13,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +28: [2023-04-29 09:45:13,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +55: [2023-04-29 09:45:13,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:13,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +37: [2023-04-29 09:45:13,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +59: [2023-04-29 09:45:13,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:13,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +21: [2023-04-29 09:45:13,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +40: [2023-04-29 09:45:13,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +16: [2023-04-29 09:45:13,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +38: [2023-04-29 09:45:13,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:13,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +17: [2023-04-29 09:45:13,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +17: [2023-04-29 09:45:13,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +25: [2023-04-29 09:45:13,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +40: [2023-04-29 09:45:13,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:13,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +47: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +21: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +36: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +12: [2023-04-29 09:45:13,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +49: [2023-04-29 09:45:13,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +17: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +27: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +21: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +62: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +62: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +27: [2023-04-29 09:45:13,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +49: [2023-04-29 09:45:13,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +19: [2023-04-29 09:45:13,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +43: [2023-04-29 09:45:13,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +30: [2023-04-29 09:45:13,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +30: [2023-04-29 09:45:13,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +12: [2023-04-29 09:45:13,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:13,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +43: [2023-04-29 09:45:13,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +12: [2023-04-29 09:45:13,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +49: [2023-04-29 09:45:13,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +30: [2023-04-29 09:45:13,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +25: [2023-04-29 09:45:13,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +49: [2023-04-29 09:45:13,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +12: [2023-04-29 09:45:13,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +62: [2023-04-29 09:45:13,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +21: [2023-04-29 09:45:13,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +44: [2023-04-29 09:45:13,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +30: [2023-04-29 09:45:13,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +19: [2023-04-29 09:45:13,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +44: [2023-04-29 09:45:13,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +27: [2023-04-29 09:45:13,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +12: [2023-04-29 09:45:13,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +16: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +25: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +21: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +21: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +25: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +36: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +19: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +37: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +37: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +44: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:13,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +49: [2023-04-29 09:45:13,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +27: [2023-04-29 09:45:13,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +61: [2023-04-29 09:45:13,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +45: [2023-04-29 09:45:13,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +26: [2023-04-29 09:45:13,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +18: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +48: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +12: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +48: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:13,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +19: [2023-04-29 09:45:13,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +44: [2023-04-29 09:45:13,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +26: [2023-04-29 09:45:13,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +45: [2023-04-29 09:45:13,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +45: [2023-04-29 09:45:13,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +18: [2023-04-29 09:45:13,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +47: [2023-04-29 09:45:13,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +37: [2023-04-29 09:45:13,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +52: [2023-04-29 09:45:13,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +48: [2023-04-29 09:45:13,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +37: [2023-04-29 09:45:13,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 0: [2023-04-29 09:45:13,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 0: [2023-04-29 09:45:13,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +37: [2023-04-29 09:45:13,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 0: [2023-04-29 09:45:13,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 0: [2023-04-29 09:45:13,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +48: [2023-04-29 09:45:13,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +26: [2023-04-29 09:45:13,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +62: [2023-04-29 09:45:13,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +21: [2023-04-29 09:45:13,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +50: [2023-04-29 09:45:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +26: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +52: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +48: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +48: [2023-04-29 09:45:13,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +20: [2023-04-29 09:45:13,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +16: [2023-04-29 09:45:13,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +26: [2023-04-29 09:45:13,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +52: [2023-04-29 09:45:13,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +37: [2023-04-29 09:45:13,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +29: [2023-04-29 09:45:13,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +20: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +49: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +20: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +26: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:13,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +50: [2023-04-29 09:45:13,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +35: [2023-04-29 09:45:13,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +26: [2023-04-29 09:45:13,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +29: [2023-04-29 09:45:13,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +49: [2023-04-29 09:45:13,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +16: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +50: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_16-model_01-model_states.pt. +10: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +16: [2023-04-29 09:45:13,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:13,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +50: [2023-04-29 09:45:13,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +49: [2023-04-29 09:45:13,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +49: [2023-04-29 09:45:13,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +19: [2023-04-29 09:45:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +35: [2023-04-29 09:45:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +26: [2023-04-29 09:45:13,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +47: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +62: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +10: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +49: [2023-04-29 09:45:13,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +30: [2023-04-29 09:45:13,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +62: [2023-04-29 09:45:13,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:13,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +35: [2023-04-29 09:45:13,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:13,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +19: [2023-04-29 09:45:13,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +49: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +35: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +51: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +15: [2023-04-29 09:45:13,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +30: [2023-04-29 09:45:13,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +37: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +14: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +37: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +37: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +45: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +16: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +40: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +60: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +31: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +60: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +51: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +22: [2023-04-29 09:45:13,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +15: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +60: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +14: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +35: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +35: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +31: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +31: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +19: [2023-04-29 09:45:13,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +15: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +51: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +35: [2023-04-29 09:45:13,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +15: [2023-04-29 09:45:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +37: [2023-04-29 09:45:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +15: [2023-04-29 09:45:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +36: [2023-04-29 09:45:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +37: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +29: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +32: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +37: [2023-04-29 09:45:13,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +26: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +37: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +58: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +31: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +19: [2023-04-29 09:45:13,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +15: [2023-04-29 09:45:13,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 0: [2023-04-29 09:45:13,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +45: [2023-04-29 09:45:13,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +40: [2023-04-29 09:45:13,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +40: [2023-04-29 09:45:13,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:13,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:13,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +31: [2023-04-29 09:45:13,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +31: [2023-04-29 09:45:13,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +29: [2023-04-29 09:45:13,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:13,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +50: [2023-04-29 09:45:13,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:13,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:13,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +10: [2023-04-29 09:45:13,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +45: [2023-04-29 09:45:13,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:13,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +29: [2023-04-29 09:45:13,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +10: [2023-04-29 09:45:13,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +28: [2023-04-29 09:45:13,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +20: [2023-04-29 09:45:13,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +35: [2023-04-29 09:45:13,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +55: [2023-04-29 09:45:13,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +55: [2023-04-29 09:45:13,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +29: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +59: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +28: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +55: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +29: [2023-04-29 09:45:13,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +52: [2023-04-29 09:45:13,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:13,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +20: [2023-04-29 09:45:13,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +52: [2023-04-29 09:45:13,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:13,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +28: [2023-04-29 09:45:13,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +56: [2023-04-29 09:45:13,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +50: [2023-04-29 09:45:13,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +16: [2023-04-29 09:45:13,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +20: [2023-04-29 09:45:13,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +26: [2023-04-29 09:45:13,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +50: [2023-04-29 09:45:13,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +28: [2023-04-29 09:45:13,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +58: [2023-04-29 09:45:13,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +26: [2023-04-29 09:45:13,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +63: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +63: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +63: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +63: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +22: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +29: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:13,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +28: [2023-04-29 09:45:13,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt... +32: [2023-04-29 09:45:13,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +26: [2023-04-29 09:45:13,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +28: [2023-04-29 09:45:13,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +22: [2023-04-29 09:45:13,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:13,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +58: [2023-04-29 09:45:13,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +17: [2023-04-29 09:45:13,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +14: [2023-04-29 09:45:13,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +14: [2023-04-29 09:45:13,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +57: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +57: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +25: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +39: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +28: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +39: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +39: [2023-04-29 09:45:13,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +22: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +59: [2023-04-29 09:45:13,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +21: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +32: [2023-04-29 09:45:13,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:13,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +21: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +58: [2023-04-29 09:45:13,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:13,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:13,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +45: [2023-04-29 09:45:13,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:13,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:13,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +26: [2023-04-29 09:45:13,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +27: [2023-04-29 09:45:13,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +17: [2023-04-29 09:45:13,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +29: [2023-04-29 09:45:13,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +56: [2023-04-29 09:45:13,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:13,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +17: [2023-04-29 09:45:13,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +25: [2023-04-29 09:45:13,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +14: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +32: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +11: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +60: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +59: [2023-04-29 09:45:13,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +27: [2023-04-29 09:45:13,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +57: [2023-04-29 09:45:13,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +57: [2023-04-29 09:45:13,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +57: [2023-04-29 09:45:13,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +11: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +56: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +17: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +11: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +11: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +40: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +46: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +46: [2023-04-29 09:45:13,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +12: [2023-04-29 09:45:13,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +46: [2023-04-29 09:45:13,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +46: [2023-04-29 09:45:13,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +27: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +14: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +33: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +33: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +33: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +40: [2023-04-29 09:45:13,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +25: [2023-04-29 09:45:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +33: [2023-04-29 09:45:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +27: [2023-04-29 09:45:13,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +17: [2023-04-29 09:45:13,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:13,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +10: [2023-04-29 09:45:13,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +44: [2023-04-29 09:45:13,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +44: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +17: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +12: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +33: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +59: [2023-04-29 09:45:13,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +12: [2023-04-29 09:45:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +54: [2023-04-29 09:45:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +54: [2023-04-29 09:45:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +27: [2023-04-29 09:45:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +48: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +48: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +10: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +27: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +33: [2023-04-29 09:45:13,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +12: [2023-04-29 09:45:13,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +54: [2023-04-29 09:45:13,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +18: [2023-04-29 09:45:13,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +44: [2023-04-29 09:45:13,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +49: [2023-04-29 09:45:13,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +44: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +25: [2023-04-29 09:45:13,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +58: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +55: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +48: [2023-04-29 09:45:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +22: [2023-04-29 09:45:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +48: [2023-04-29 09:45:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:13,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +48: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +48: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +12: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +55: [2023-04-29 09:45:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +55: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +40: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +55: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +18: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +50: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +25: [2023-04-29 09:45:13,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +26: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +44: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +12: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +27: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +26: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +58: [2023-04-29 09:45:13,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:13,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:13,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +22: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +18: [2023-04-29 09:45:13,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +18: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +51: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +25: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +49: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +39: [2023-04-29 09:45:13,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +26: [2023-04-29 09:45:13,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +63: [2023-04-29 09:45:13,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +39: [2023-04-29 09:45:13,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +22: [2023-04-29 09:45:13,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +58: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +63: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +14: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +63: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +61: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +61: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +61: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +61: [2023-04-29 09:45:13,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +30: [2023-04-29 09:45:13,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +59: [2023-04-29 09:45:13,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:13,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +25: [2023-04-29 09:45:13,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +35: [2023-04-29 09:45:13,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +21: [2023-04-29 09:45:13,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:13,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +24: [2023-04-29 09:45:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:13,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +46: [2023-04-29 09:45:13,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +59: [2023-04-29 09:45:13,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +30: [2023-04-29 09:45:13,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +62: [2023-04-29 09:45:13,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +50: [2023-04-29 09:45:13,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +38: [2023-04-29 09:45:13,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +38: [2023-04-29 09:45:13,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +24: [2023-04-29 09:45:13,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +46: [2023-04-29 09:45:13,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +46: [2023-04-29 09:45:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +24: [2023-04-29 09:45:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:13,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +47: [2023-04-29 09:45:13,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +38: [2023-04-29 09:45:13,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +49: [2023-04-29 09:45:13,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +24: [2023-04-29 09:45:13,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +35: [2023-04-29 09:45:13,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_38-model_01-model_states.pt. +24: [2023-04-29 09:45:13,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +15: [2023-04-29 09:45:13,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +14: [2023-04-29 09:45:13,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +50: [2023-04-29 09:45:13,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:13,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +62: [2023-04-29 09:45:13,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +51: [2023-04-29 09:45:13,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +11: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +11: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +11: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +11: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +59: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +15: [2023-04-29 09:45:13,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +14: [2023-04-29 09:45:13,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +30: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +30: [2023-04-29 09:45:13,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +15: [2023-04-29 09:45:13,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +49: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +15: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +51: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +37: [2023-04-29 09:45:13,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:13,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:13,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:13,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:13,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +50: [2023-04-29 09:45:13,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:13,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +54: [2023-04-29 09:45:13,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +15: [2023-04-29 09:45:13,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +44: [2023-04-29 09:45:13,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +37: [2023-04-29 09:45:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:13,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +15: [2023-04-29 09:45:13,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +30: [2023-04-29 09:45:13,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +30: [2023-04-29 09:45:13,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +30: [2023-04-29 09:45:13,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +54: [2023-04-29 09:45:13,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +54: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +49: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +56: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +44: [2023-04-29 09:45:13,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +56: [2023-04-29 09:45:13,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +35: [2023-04-29 09:45:13,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:13,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +44: [2023-04-29 09:45:13,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +44: [2023-04-29 09:45:13,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +40: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +20: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:13,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:13,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +56: [2023-04-29 09:45:13,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +56: [2023-04-29 09:45:13,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:13,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +52: [2023-04-29 09:45:13,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +56: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +57: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +57: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +20: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 0: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +20: [2023-04-29 09:45:13,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +61: [2023-04-29 09:45:13,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +60: [2023-04-29 09:45:13,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +52: [2023-04-29 09:45:13,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +40: [2023-04-29 09:45:13,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +52: [2023-04-29 09:45:13,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +47: [2023-04-29 09:45:13,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +57: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +57: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +25: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +25: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +25: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +25: [2023-04-29 09:45:13,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +45: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +45: [2023-04-29 09:45:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +45: [2023-04-29 09:45:13,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +45: [2023-04-29 09:45:13,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +35: [2023-04-29 09:45:13,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:13,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +58: [2023-04-29 09:45:13,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_00-model_states.pt. +58: [2023-04-29 09:45:13,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +33: [2023-04-29 09:45:13,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +58: [2023-04-29 09:45:13,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +52: [2023-04-29 09:45:13,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +52: [2023-04-29 09:45:13,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +60: [2023-04-29 09:45:13,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +35: [2023-04-29 09:45:13,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:13,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +47: [2023-04-29 09:45:13,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt... +38: [2023-04-29 09:45:13,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:13,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +25: [2023-04-29 09:45:13,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +49: [2023-04-29 09:45:13,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +20: [2023-04-29 09:45:13,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:13,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +25: [2023-04-29 09:45:13,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +42: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +42: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +60: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +42: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +11: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +11: [2023-04-29 09:45:13,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +48: [2023-04-29 09:45:13,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +11: [2023-04-29 09:45:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +37: [2023-04-29 09:45:13,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt... +49: [2023-04-29 09:45:13,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:13,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +11: [2023-04-29 09:45:13,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +51: [2023-04-29 09:45:13,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +51: [2023-04-29 09:45:13,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +53: [2023-04-29 09:45:13,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +53: [2023-04-29 09:45:13,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +53: [2023-04-29 09:45:13,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +53: [2023-04-29 09:45:13,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +60: [2023-04-29 09:45:13,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +20: [2023-04-29 09:45:13,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +37: [2023-04-29 09:45:13,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:13,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:13,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:13,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +39: [2023-04-29 09:45:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:13,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +48: [2023-04-29 09:45:13,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:13,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +39: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +39: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +63: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +40: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:13,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +55: [2023-04-29 09:45:13,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +34: [2023-04-29 09:45:13,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:13,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:13,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:13,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +14: [2023-04-29 09:45:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +45: [2023-04-29 09:45:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +58: [2023-04-29 09:45:13,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +58: [2023-04-29 09:45:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +14: [2023-04-29 09:45:13,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +45: [2023-04-29 09:45:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:13,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:13,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +33: [2023-04-29 09:45:13,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +58: [2023-04-29 09:45:13,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:13,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:13,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +45: [2023-04-29 09:45:13,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:13,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +46: [2023-04-29 09:45:13,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +39: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +55: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +55: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +60: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:13,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +14: [2023-04-29 09:45:13,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +13: [2023-04-29 09:45:13,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +22: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +63: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +15: [2023-04-29 09:45:13,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +33: [2023-04-29 09:45:13,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:13,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +53: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +39: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +22: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +60: [2023-04-29 09:45:13,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +22: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +62: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +62: [2023-04-29 09:45:13,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +62: [2023-04-29 09:45:13,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +46: [2023-04-29 09:45:13,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +62: [2023-04-29 09:45:13,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +63: [2023-04-29 09:45:13,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +42: [2023-04-29 09:45:13,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:13,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +13: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +55: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +55: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +53: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +13: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +53: [2023-04-29 09:45:13,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:13,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:13,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +48: [2023-04-29 09:45:13,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +23: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +23: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +62: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +23: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +46: [2023-04-29 09:45:13,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +62: [2023-04-29 09:45:13,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +53: [2023-04-29 09:45:13,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +24: [2023-04-29 09:45:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +62: [2023-04-29 09:45:13,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:13,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +23: [2023-04-29 09:45:13,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +46: [2023-04-29 09:45:13,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +14: [2023-04-29 09:45:13,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +10: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +10: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +11: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +48: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +11: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +33: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +10: [2023-04-29 09:45:13,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +10: [2023-04-29 09:45:13,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +15: [2023-04-29 09:45:13,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +46: [2023-04-29 09:45:13,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +54: [2023-04-29 09:45:13,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +14: [2023-04-29 09:45:13,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +56: [2023-04-29 09:45:13,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +61: [2023-04-29 09:45:13,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +24: [2023-04-29 09:45:13,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +56: [2023-04-29 09:45:13,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +54: [2023-04-29 09:45:13,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:13,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +54: [2023-04-29 09:45:13,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +47: [2023-04-29 09:45:13,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +22: [2023-04-29 09:45:13,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +15: [2023-04-29 09:45:13,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:13,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +22: [2023-04-29 09:45:13,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +34: [2023-04-29 09:45:13,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +34: [2023-04-29 09:45:13,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +34: [2023-04-29 09:45:13,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:13,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +22: [2023-04-29 09:45:13,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +11: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +14: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +32: [2023-04-29 09:45:13,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +32: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +32: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +54: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +61: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +61: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +32: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +57: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +57: [2023-04-29 09:45:13,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:13,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +38: [2023-04-29 09:45:13,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +25: [2023-04-29 09:45:13,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +41: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +47: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +41: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +41: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +41: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +53: [2023-04-29 09:45:13,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:13,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:13,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +15: [2023-04-29 09:45:13,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:13,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +38: [2023-04-29 09:45:13,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +53: [2023-04-29 09:45:13,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +53: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +61: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +47: [2023-04-29 09:45:13,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +54: [2023-04-29 09:45:13,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +53: [2023-04-29 09:45:13,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +11: [2023-04-29 09:45:13,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +10: [2023-04-29 09:45:13,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +47: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_00-model_states.pt. +38: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +47: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +25: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +25: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +38: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +53: [2023-04-29 09:45:13,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +10: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +24: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +24: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +58: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +38: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +10: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +54: [2023-04-29 09:45:13,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +18: [2023-04-29 09:45:13,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +56: [2023-04-29 09:45:13,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +19: [2023-04-29 09:45:13,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +10: [2023-04-29 09:45:13,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +15: [2023-04-29 09:45:13,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:13,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +54: [2023-04-29 09:45:13,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +54: [2023-04-29 09:45:13,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:13,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt... +25: [2023-04-29 09:45:13,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +18: [2023-04-29 09:45:13,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +42: [2023-04-29 09:45:13,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:13,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +18: [2023-04-29 09:45:13,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +32: [2023-04-29 09:45:13,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +18: [2023-04-29 09:45:13,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +45: [2023-04-29 09:45:13,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:13,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +58: [2023-04-29 09:45:13,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +43: [2023-04-29 09:45:13,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +43: [2023-04-29 09:45:13,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +19: [2023-04-29 09:45:13,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +19: [2023-04-29 09:45:13,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +43: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +19: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +57: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:13,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +19: [2023-04-29 09:45:13,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +11: [2023-04-29 09:45:13,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:13,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:13,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +32: [2023-04-29 09:45:13,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +25: [2023-04-29 09:45:13,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +63: [2023-04-29 09:45:13,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +41: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +57: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +58: [2023-04-29 09:45:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 3: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +17: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +20: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:13,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +20: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +63: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +45: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +11: [2023-04-29 09:45:13,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +11: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +42: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +58: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +58: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +28: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:13,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:13,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +42: [2023-04-29 09:45:13,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:13,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +23: [2023-04-29 09:45:13,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +60: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +17: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +62: [2023-04-29 09:45:13,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +17: [2023-04-29 09:45:13,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +28: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +28: [2023-04-29 09:45:13,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +20: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +45: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +45: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +45: [2023-04-29 09:45:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +58: [2023-04-29 09:45:13,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +62: [2023-04-29 09:45:13,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +29: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +34: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +23: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +42: [2023-04-29 09:45:13,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +46: [2023-04-29 09:45:13,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +46: [2023-04-29 09:45:13,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +28: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +46: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +46: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +28: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +62: [2023-04-29 09:45:13,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +24: [2023-04-29 09:45:13,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +24: [2023-04-29 09:45:13,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +24: [2023-04-29 09:45:13,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +24: [2023-04-29 09:45:13,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +62: [2023-04-29 09:45:13,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +29: [2023-04-29 09:45:13,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +42: [2023-04-29 09:45:13,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:13,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:13,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +46: [2023-04-29 09:45:13,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +16: [2023-04-29 09:45:13,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +16: [2023-04-29 09:45:13,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +46: [2023-04-29 09:45:13,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:13,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:13,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +53: [2023-04-29 09:45:13,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +42: [2023-04-29 09:45:13,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +16: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +16: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +42: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +23: [2023-04-29 09:45:13,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +47: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +33: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +33: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +33: [2023-04-29 09:45:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +33: [2023-04-29 09:45:13,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +22: [2023-04-29 09:45:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +53: [2023-04-29 09:45:13,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +63: [2023-04-29 09:45:13,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +13: [2023-04-29 09:45:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +23: [2023-04-29 09:45:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +43: [2023-04-29 09:45:13,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +47: [2023-04-29 09:45:13,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +43: [2023-04-29 09:45:13,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +43: [2023-04-29 09:45:13,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +59: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +42: [2023-04-29 09:45:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +59: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +59: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +59: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +36: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +36: [2023-04-29 09:45:13,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +63: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +13: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +13: [2023-04-29 09:45:13,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +47: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +62: [2023-04-29 09:45:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +62: [2023-04-29 09:45:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +51: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +51: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +39: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +39: [2023-04-29 09:45:13,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +39: [2023-04-29 09:45:13,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:13,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +61: [2023-04-29 09:45:13,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +61: [2023-04-29 09:45:13,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +22: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +22: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +61: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +39: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +60: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +22: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +30: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +61: [2023-04-29 09:45:13,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +52: [2023-04-29 09:45:13,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +31: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +61: [2023-04-29 09:45:13,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +22: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +30: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +30: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +54: [2023-04-29 09:45:13,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +30: [2023-04-29 09:45:13,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +62: [2023-04-29 09:45:13,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +24: [2023-04-29 09:45:13,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:13,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +10: [2023-04-29 09:45:13,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +53: [2023-04-29 09:45:13,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:13,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +53: [2023-04-29 09:45:13,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:13,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +52: [2023-04-29 09:45:13,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:13,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:13,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +18: [2023-04-29 09:45:13,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +16: [2023-04-29 09:45:13,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +16: [2023-04-29 09:45:13,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +16: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +16: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:13,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:13,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +59: [2023-04-29 09:45:13,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:13,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +41: [2023-04-29 09:45:13,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:13,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:13,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +41: [2023-04-29 09:45:13,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +10: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +38: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +38: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +33: [2023-04-29 09:45:13,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +36: [2023-04-29 09:45:13,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:13,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:13,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +34: [2023-04-29 09:45:13,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +33: [2023-04-29 09:45:13,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:13,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:13,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +41: [2023-04-29 09:45:13,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:13,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:13,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +36: [2023-04-29 09:45:13,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +38: [2023-04-29 09:45:13,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:13,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:13,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:13,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +34: [2023-04-29 09:45:13,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +36: [2023-04-29 09:45:13,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:13,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +54: [2023-04-29 09:45:13,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +42: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +42: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:13,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +17: [2023-04-29 09:45:13,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +32: [2023-04-29 09:45:13,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +51: [2023-04-29 09:45:13,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +28: [2023-04-29 09:45:13,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:13,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +18: [2023-04-29 09:45:13,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:13,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +55: [2023-04-29 09:45:13,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +32: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +20: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +51: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +28: [2023-04-29 09:45:13,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +34: [2023-04-29 09:45:13,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +41: [2023-04-29 09:45:13,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +17: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +43: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +19: [2023-04-29 09:45:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:13,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +53: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +32: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +34: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +29: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:13,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +55: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +43: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +43: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +41: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +32: [2023-04-29 09:45:13,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +27: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +29: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:13,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +36: [2023-04-29 09:45:13,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +12: [2023-04-29 09:45:13,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +12: [2023-04-29 09:45:13,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 0: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +36: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +41: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +29: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +53: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:13,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +36: [2023-04-29 09:45:13,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:13,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +28: [2023-04-29 09:45:13,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:13,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +43: [2023-04-29 09:45:13,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +18: [2023-04-29 09:45:13,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +27: [2023-04-29 09:45:13,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:13,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +20: [2023-04-29 09:45:13,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:13,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:13,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +13: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +46: [2023-04-29 09:45:13,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +12: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +23: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +23: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +23: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +12: [2023-04-29 09:45:13,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +42: [2023-04-29 09:45:13,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +18: [2023-04-29 09:45:13,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +13: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +13: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +17: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +46: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +46: [2023-04-29 09:45:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +34: [2023-04-29 09:45:13,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +47: [2023-04-29 09:45:13,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +42: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +16: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:13,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:13,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +54: [2023-04-29 09:45:13,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:13,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +42: [2023-04-29 09:45:13,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +28: [2023-04-29 09:45:13,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:13,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:13,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +61: [2023-04-29 09:45:13,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +18: [2023-04-29 09:45:13,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +47: [2023-04-29 09:45:13,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +16: [2023-04-29 09:45:13,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:13,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:13,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:13,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:13,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +16: [2023-04-29 09:45:13,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +54: [2023-04-29 09:45:13,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:13,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:13,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:13,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +39: [2023-04-29 09:45:13,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +19: [2023-04-29 09:45:13,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +29: [2023-04-29 09:45:13,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:13,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:13,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:13,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:13,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +43: [2023-04-29 09:45:13,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:13,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +24: [2023-04-29 09:45:13,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +19: [2023-04-29 09:45:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:13,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +29: [2023-04-29 09:45:13,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:13,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:13,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +43: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +29: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +24: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +61: [2023-04-29 09:45:13,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:13,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +55: [2023-04-29 09:45:13,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +17: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +43: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:13,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +13: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +49: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +49: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +31: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +12: [2023-04-29 09:45:13,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +12: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +12: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +46: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +30: [2023-04-29 09:45:13,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +43: [2023-04-29 09:45:13,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +61: [2023-04-29 09:45:13,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +61: [2023-04-29 09:45:13,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +49: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +59: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +49: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +59: [2023-04-29 09:45:13,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:13,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:13,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +23: [2023-04-29 09:45:13,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +24: [2023-04-29 09:45:13,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +20: [2023-04-29 09:45:13,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +42: [2023-04-29 09:45:13,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:13,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +43: [2023-04-29 09:45:13,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:13,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +13: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +35: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +35: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +35: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +24: [2023-04-29 09:45:13,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +35: [2023-04-29 09:45:13,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:13,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +59: [2023-04-29 09:45:13,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +52: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +16: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +16: [2023-04-29 09:45:13,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +59: [2023-04-29 09:45:13,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +48: [2023-04-29 09:45:13,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +48: [2023-04-29 09:45:13,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +48: [2023-04-29 09:45:13,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +48: [2023-04-29 09:45:13,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +59: [2023-04-29 09:45:13,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +24: [2023-04-29 09:45:13,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +39: [2023-04-29 09:45:13,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +47: [2023-04-29 09:45:13,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:13,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +41: [2023-04-29 09:45:13,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +41: [2023-04-29 09:45:13,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +38: [2023-04-29 09:45:13,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +59: [2023-04-29 09:45:13,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +16: [2023-04-29 09:45:13,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +16: [2023-04-29 09:45:13,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +33: [2023-04-29 09:45:13,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +20: [2023-04-29 09:45:13,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +33: [2023-04-29 09:45:13,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +33: [2023-04-29 09:45:13,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +50: [2023-04-29 09:45:13,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +39: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +38: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +50: [2023-04-29 09:45:13,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +16: [2023-04-29 09:45:13,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +29: [2023-04-29 09:45:13,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +28: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +28: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +28: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +28: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +59: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +37: [2023-04-29 09:45:13,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +37: [2023-04-29 09:45:13,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 1: [2023-04-29 09:45:13,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:13,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:13,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +36: [2023-04-29 09:45:13,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +16: [2023-04-29 09:45:13,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +36: [2023-04-29 09:45:13,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +61: [2023-04-29 09:45:13,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +59: [2023-04-29 09:45:13,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +37: [2023-04-29 09:45:13,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +37: [2023-04-29 09:45:13,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +30: [2023-04-29 09:45:13,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:13,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:13,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +33: [2023-04-29 09:45:13,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +33: [2023-04-29 09:45:13,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +45: [2023-04-29 09:45:13,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +45: [2023-04-29 09:45:13,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +31: [2023-04-29 09:45:13,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +45: [2023-04-29 09:45:13,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +45: [2023-04-29 09:45:13,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:13,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:13,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +39: [2023-04-29 09:45:13,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +30: [2023-04-29 09:45:13,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 5: [2023-04-29 09:45:13,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +27: [2023-04-29 09:45:13,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +51: [2023-04-29 09:45:13,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +52: [2023-04-29 09:45:13,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +41: [2023-04-29 09:45:13,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +16: [2023-04-29 09:45:13,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +12: [2023-04-29 09:45:13,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:13,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:13,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +61: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +36: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +49: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +49: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +49: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +59: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +59: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +30: [2023-04-29 09:45:13,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:13,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +19: [2023-04-29 09:45:13,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +12: [2023-04-29 09:45:13,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:13,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +45: [2023-04-29 09:45:13,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +45: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +61: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +45: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +45: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 1: [2023-04-29 09:45:13,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +35: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +30: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +51: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +35: [2023-04-29 09:45:13,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +19: [2023-04-29 09:45:13,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +49: [2023-04-29 09:45:13,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +48: [2023-04-29 09:45:13,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +52: [2023-04-29 09:45:13,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +51: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +43: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 1: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +26: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +26: [2023-04-29 09:45:13,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +26: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +36: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 8: [2023-04-29 09:45:13,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +27: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +41: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +41: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +32: [2023-04-29 09:45:13,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +58: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +58: [2023-04-29 09:45:13,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +58: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +56: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +56: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +56: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +58: [2023-04-29 09:45:13,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +36: [2023-04-29 09:45:13,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:13,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +27: [2023-04-29 09:45:13,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +48: [2023-04-29 09:45:13,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +51: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +38: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:13,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +37: [2023-04-29 09:45:13,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:13,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:13,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +38: [2023-04-29 09:45:13,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:13,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +43: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +41: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:13,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:13,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:13,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:13,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:13,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +28: [2023-04-29 09:45:13,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +27: [2023-04-29 09:45:13,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:13,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +12: [2023-04-29 09:45:13,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:13,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +20: [2023-04-29 09:45:13,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +20: [2023-04-29 09:45:13,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +20: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +50: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +28: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +28: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +50: [2023-04-29 09:45:13,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +50: [2023-04-29 09:45:13,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +32: [2023-04-29 09:45:13,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +12: [2023-04-29 09:45:13,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:13,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:13,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:13,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +19: [2023-04-29 09:45:13,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:13,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +13: [2023-04-29 09:45:13,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +32: [2023-04-29 09:45:13,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +19: [2023-04-29 09:45:13,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:13,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:13,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:13,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:13,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +23: [2023-04-29 09:45:13,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +23: [2023-04-29 09:45:13,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:13,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +16: [2023-04-29 09:45:13,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:13,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:13,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:13,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:13,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:13,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:13,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:13,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +27: [2023-04-29 09:45:13,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:13,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +27: [2023-04-29 09:45:13,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:13,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:13,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:13,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +13: [2023-04-29 09:45:13,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +14: [2023-04-29 09:45:13,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:13,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +51: [2023-04-29 09:45:13,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 0: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 0: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 0: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +56: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +27: [2023-04-29 09:45:13,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +56: [2023-04-29 09:45:13,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +23: [2023-04-29 09:45:13,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +23: [2023-04-29 09:45:13,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +59: [2023-04-29 09:45:13,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:13,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:13,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +26: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +51: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +49: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +57: [2023-04-29 09:45:13,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +56: [2023-04-29 09:45:13,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +12: [2023-04-29 09:45:13,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +57: [2023-04-29 09:45:13,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +43: [2023-04-29 09:45:13,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +57: [2023-04-29 09:45:13,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +14: [2023-04-29 09:45:13,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +14: [2023-04-29 09:45:13,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +14: [2023-04-29 09:45:13,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +56: [2023-04-29 09:45:13,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +13: [2023-04-29 09:45:13,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +14: [2023-04-29 09:45:13,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +59: [2023-04-29 09:45:13,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +23: [2023-04-29 09:45:13,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +23: [2023-04-29 09:45:13,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +21: [2023-04-29 09:45:13,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +21: [2023-04-29 09:45:13,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +21: [2023-04-29 09:45:13,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +21: [2023-04-29 09:45:13,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +20: [2023-04-29 09:45:13,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +44: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +44: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +44: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +16: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +44: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +13: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +49: [2023-04-29 09:45:13,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +49: [2023-04-29 09:45:13,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +49: [2023-04-29 09:45:13,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:13,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +59: [2023-04-29 09:45:13,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +40: [2023-04-29 09:45:13,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +40: [2023-04-29 09:45:13,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +32: [2023-04-29 09:45:13,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:13,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:13,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:13,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +11: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +11: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +40: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +11: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +11: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +32: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:13,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +45: [2023-04-29 09:45:13,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +12: [2023-04-29 09:45:13,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:13,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +35: [2023-04-29 09:45:13,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +32: [2023-04-29 09:45:13,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:13,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +49: [2023-04-29 09:45:13,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +49: [2023-04-29 09:45:13,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +52: [2023-04-29 09:45:13,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +52: [2023-04-29 09:45:13,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +16: [2023-04-29 09:45:13,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +35: [2023-04-29 09:45:13,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +50: [2023-04-29 09:45:13,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:13,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:13,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +14: [2023-04-29 09:45:13,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +35: [2023-04-29 09:45:13,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +40: [2023-04-29 09:45:13,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +35: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +37: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +60: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +57: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +16: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +12: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 9: [2023-04-29 09:45:13,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +59: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +49: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +49: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +49: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +45: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +45: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +45: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +21: [2023-04-29 09:45:13,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +57: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +58: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +31: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +31: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +15: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +15: [2023-04-29 09:45:13,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +15: [2023-04-29 09:45:13,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +40: [2023-04-29 09:45:13,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +15: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +37: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +40: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:13,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:13,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +50: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +35: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +57: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +31: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +35: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +12: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +62: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +62: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +35: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +62: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +12: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +62: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +60: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +44: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +44: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +44: [2023-04-29 09:45:13,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +44: [2023-04-29 09:45:13,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +21: [2023-04-29 09:45:13,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +21: [2023-04-29 09:45:13,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:13,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +40: [2023-04-29 09:45:13,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:13,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:13,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +58: [2023-04-29 09:45:13,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +58: [2023-04-29 09:45:13,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +12: [2023-04-29 09:45:13,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +40: [2023-04-29 09:45:13,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:13,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:13,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +11: [2023-04-29 09:45:13,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +12: [2023-04-29 09:45:13,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +40: [2023-04-29 09:45:13,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +12: [2023-04-29 09:45:13,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +49: [2023-04-29 09:45:13,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +11: [2023-04-29 09:45:13,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +50: [2023-04-29 09:45:13,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:13,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:13,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +48: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +11: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +11: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +25: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 6: [2023-04-29 09:45:13,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +45: [2023-04-29 09:45:13,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +37: [2023-04-29 09:45:13,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +17: [2023-04-29 09:45:13,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +17: [2023-04-29 09:45:13,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +40: [2023-04-29 09:45:13,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +48: [2023-04-29 09:45:13,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +58: [2023-04-29 09:45:13,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:13,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +12: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +37: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +25: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +59: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +25: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 7: [2023-04-29 09:45:13,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +50: [2023-04-29 09:45:13,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +17: [2023-04-29 09:45:13,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +17: [2023-04-29 09:45:13,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. + 5: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 5: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +27: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +27: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +27: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +12: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +40: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +27: [2023-04-29 09:45:13,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +48: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +37: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +35: [2023-04-29 09:45:13,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:13,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +48: [2023-04-29 09:45:13,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 5: [2023-04-29 09:45:13,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +28: [2023-04-29 09:45:13,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +44: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +44: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +37: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +29: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +29: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +29: [2023-04-29 09:45:13,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +52: [2023-04-29 09:45:13,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +29: [2023-04-29 09:45:13,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +51: [2023-04-29 09:45:13,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +35: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +47: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +47: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +35: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +46: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +19: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +44: [2023-04-29 09:45:13,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 3: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +48: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +44: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +35: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +37: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +19: [2023-04-29 09:45:13,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +28: [2023-04-29 09:45:13,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +52: [2023-04-29 09:45:13,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 7: [2023-04-29 09:45:13,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 5: [2023-04-29 09:45:13,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +15: [2023-04-29 09:45:13,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +26: [2023-04-29 09:45:13,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +44: [2023-04-29 09:45:13,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +45: [2023-04-29 09:45:13,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +15: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +44: [2023-04-29 09:45:13,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +15: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +44: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +15: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +60: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +19: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +58: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:13,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +55: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +52: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +55: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +31: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +56: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +31: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +31: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +46: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +19: [2023-04-29 09:45:13,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +46: [2023-04-29 09:45:13,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +22: [2023-04-29 09:45:13,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +22: [2023-04-29 09:45:13,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +31: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +58: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +26: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +60: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +50: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +45: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +26: [2023-04-29 09:45:13,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +51: [2023-04-29 09:45:13,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:13,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +14: [2023-04-29 09:45:13,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +22: [2023-04-29 09:45:13,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +28: [2023-04-29 09:45:13,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +51: [2023-04-29 09:45:13,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:13,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +50: [2023-04-29 09:45:13,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 5: [2023-04-29 09:45:13,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +19: [2023-04-29 09:45:13,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +62: [2023-04-29 09:45:13,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 4: [2023-04-29 09:45:13,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +60: [2023-04-29 09:45:13,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +26: [2023-04-29 09:45:13,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +56: [2023-04-29 09:45:13,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 4: [2023-04-29 09:45:13,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +26: [2023-04-29 09:45:13,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +19: [2023-04-29 09:45:13,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +56: [2023-04-29 09:45:13,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +49: [2023-04-29 09:45:13,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:13,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +26: [2023-04-29 09:45:13,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +50: [2023-04-29 09:45:13,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +27: [2023-04-29 09:45:13,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +19: [2023-04-29 09:45:13,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +17: [2023-04-29 09:45:13,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +17: [2023-04-29 09:45:13,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +50: [2023-04-29 09:45:13,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +10: [2023-04-29 09:45:13,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +28: [2023-04-29 09:45:13,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +10: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +20: [2023-04-29 09:45:13,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 4: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +10: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +56: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +62: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +62: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +56: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +50: [2023-04-29 09:45:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +33: [2023-04-29 09:45:13,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +58: [2023-04-29 09:45:13,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +25: [2023-04-29 09:45:13,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +25: [2023-04-29 09:45:13,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +26: [2023-04-29 09:45:13,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +26: [2023-04-29 09:45:13,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +33: [2023-04-29 09:45:13,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +17: [2023-04-29 09:45:13,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +20: [2023-04-29 09:45:13,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 7: [2023-04-29 09:45:13,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +51: [2023-04-29 09:45:13,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +27: [2023-04-29 09:45:13,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +27: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... + 7: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +10: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +61: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +61: [2023-04-29 09:45:13,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +61: [2023-04-29 09:45:13,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +46: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +46: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +10: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +33: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:13,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +27: [2023-04-29 09:45:13,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +10: [2023-04-29 09:45:13,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:13,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:13,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +17: [2023-04-29 09:45:13,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +33: [2023-04-29 09:45:13,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +49: [2023-04-29 09:45:13,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +29: [2023-04-29 09:45:13,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +46: [2023-04-29 09:45:13,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:13,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:13,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +21: [2023-04-29 09:45:13,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +47: [2023-04-29 09:45:13,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +47: [2023-04-29 09:45:13,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +10: [2023-04-29 09:45:13,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +55: [2023-04-29 09:45:13,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +10: [2023-04-29 09:45:13,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +55: [2023-04-29 09:45:13,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +47: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +20: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +48: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +11: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +11: [2023-04-29 09:45:13,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:13,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +21: [2023-04-29 09:45:13,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +11: [2023-04-29 09:45:13,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +11: [2023-04-29 09:45:13,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +55: [2023-04-29 09:45:13,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +55: [2023-04-29 09:45:13,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +51: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +21: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:13,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +48: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +22: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +48: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +22: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:13,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +51: [2023-04-29 09:45:13,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +21: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +11: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +50: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:13,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +11: [2023-04-29 09:45:13,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +48: [2023-04-29 09:45:13,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +11: [2023-04-29 09:45:13,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +57: [2023-04-29 09:45:13,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +11: [2023-04-29 09:45:13,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +14: [2023-04-29 09:45:13,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +48: [2023-04-29 09:45:13,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:13,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +14: [2023-04-29 09:45:13,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +63: [2023-04-29 09:45:13,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +63: [2023-04-29 09:45:13,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +63: [2023-04-29 09:45:13,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +11: [2023-04-29 09:45:13,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +63: [2023-04-29 09:45:13,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +48: [2023-04-29 09:45:13,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +21: [2023-04-29 09:45:13,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +10: [2023-04-29 09:45:13,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:13,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +10: [2023-04-29 09:45:13,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:13,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +57: [2023-04-29 09:45:13,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +20: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +11: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +57: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +57: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +21: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +44: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +22: [2023-04-29 09:45:13,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +21: [2023-04-29 09:45:13,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +40: [2023-04-29 09:45:13,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +10: [2023-04-29 09:45:13,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:13,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +18: [2023-04-29 09:45:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +18: [2023-04-29 09:45:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +61: [2023-04-29 09:45:13,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +40: [2023-04-29 09:45:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 0: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +61: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +61: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +10: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +57: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +57: [2023-04-29 09:45:13,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +11: [2023-04-29 09:45:13,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +11: [2023-04-29 09:45:13,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +44: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +44: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +18: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +21: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +18: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +21: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +11: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +44: [2023-04-29 09:45:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +49: [2023-04-29 09:45:13,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:13,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +21: [2023-04-29 09:45:13,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +49: [2023-04-29 09:45:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +57: [2023-04-29 09:45:13,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +15: [2023-04-29 09:45:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +44: [2023-04-29 09:45:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +40: [2023-04-29 09:45:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +11: [2023-04-29 09:45:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +44: [2023-04-29 09:45:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +44: [2023-04-29 09:45:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +50: [2023-04-29 09:45:13,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:13,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:13,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +34: [2023-04-29 09:45:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +34: [2023-04-29 09:45:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +30: [2023-04-29 09:45:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +30: [2023-04-29 09:45:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +40: [2023-04-29 09:45:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +34: [2023-04-29 09:45:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +30: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +49: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +30: [2023-04-29 09:45:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_17-model_01-model_states.pt. +56: [2023-04-29 09:45:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +15: [2023-04-29 09:45:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +52: [2023-04-29 09:45:13,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +56: [2023-04-29 09:45:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:13,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:13,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:13,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +27: [2023-04-29 09:45:13,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +34: [2023-04-29 09:45:13,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +31: [2023-04-29 09:45:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +34: [2023-04-29 09:45:13,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +60: [2023-04-29 09:45:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +34: [2023-04-29 09:45:13,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +56: [2023-04-29 09:45:13,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +35: [2023-04-29 09:45:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:13,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:13,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +40: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +27: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +60: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +15: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +56: [2023-04-29 09:45:13,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +52: [2023-04-29 09:45:13,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +52: [2023-04-29 09:45:13,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +56: [2023-04-29 09:45:13,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +62: [2023-04-29 09:45:13,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +63: [2023-04-29 09:45:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +13: [2023-04-29 09:45:13,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +40: [2023-04-29 09:45:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +13: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +63: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +13: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +14: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. + 2: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 2: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +15: [2023-04-29 09:45:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +44: [2023-04-29 09:45:13,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:13,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +44: [2023-04-29 09:45:13,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +15: [2023-04-29 09:45:13,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +60: [2023-04-29 09:45:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +15: [2023-04-29 09:45:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +60: [2023-04-29 09:45:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +31: [2023-04-29 09:45:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +63: [2023-04-29 09:45:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +52: [2023-04-29 09:45:13,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +62: [2023-04-29 09:45:13,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +52: [2023-04-29 09:45:13,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +14: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +40: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +52: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 9: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +13: [2023-04-29 09:45:13,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +17: [2023-04-29 09:45:13,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +18: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +62: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +13: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +18: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +18: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +13: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +31: [2023-04-29 09:45:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +27: [2023-04-29 09:45:13,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +14: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +60: [2023-04-29 09:45:13,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +15: [2023-04-29 09:45:13,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +14: [2023-04-29 09:45:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +44: [2023-04-29 09:45:13,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +25: [2023-04-29 09:45:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +29: [2023-04-29 09:45:13,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +46: [2023-04-29 09:45:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +25: [2023-04-29 09:45:13,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +17: [2023-04-29 09:45:13,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +17: [2023-04-29 09:45:13,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +62: [2023-04-29 09:45:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +14: [2023-04-29 09:45:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +44: [2023-04-29 09:45:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +18: [2023-04-29 09:45:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +14: [2023-04-29 09:45:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +62: [2023-04-29 09:45:13,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +15: [2023-04-29 09:45:13,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +50: [2023-04-29 09:45:13,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +14: [2023-04-29 09:45:13,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +46: [2023-04-29 09:45:13,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +27: [2023-04-29 09:45:13,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +25: [2023-04-29 09:45:13,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +25: [2023-04-29 09:45:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +50: [2023-04-29 09:45:13,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +25: [2023-04-29 09:45:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +17: [2023-04-29 09:45:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +33: [2023-04-29 09:45:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +30: [2023-04-29 09:45:13,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +40: [2023-04-29 09:45:13,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:13,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +54: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +30: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +30: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +30: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt... +29: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +33: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +29: [2023-04-29 09:45:13,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +29: [2023-04-29 09:45:13,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +35: [2023-04-29 09:45:13,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +27: [2023-04-29 09:45:13,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +15: [2023-04-29 09:45:13,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +15: [2023-04-29 09:45:13,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +15: [2023-04-29 09:45:13,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +27: [2023-04-29 09:45:13,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +53: [2023-04-29 09:45:13,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +40: [2023-04-29 09:45:13,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:13,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +25: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +25: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +25: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +17: [2023-04-29 09:45:13,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +35: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +25: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +47: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +25: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +46: [2023-04-29 09:45:13,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +54: [2023-04-29 09:45:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +40: [2023-04-29 09:45:13,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +63: [2023-04-29 09:45:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +29: [2023-04-29 09:45:13,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +29: [2023-04-29 09:45:13,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +53: [2023-04-29 09:45:13,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +17: [2023-04-29 09:45:13,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +27: [2023-04-29 09:45:13,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +46: [2023-04-29 09:45:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +53: [2023-04-29 09:45:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +54: [2023-04-29 09:45:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_39-model_01-model_states.pt. +53: [2023-04-29 09:45:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +53: [2023-04-29 09:45:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +22: [2023-04-29 09:45:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +63: [2023-04-29 09:45:13,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:13,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +47: [2023-04-29 09:45:13,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +25: [2023-04-29 09:45:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +48: [2023-04-29 09:45:13,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:13,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +10: [2023-04-29 09:45:13,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +62: [2023-04-29 09:45:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +10: [2023-04-29 09:45:13,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +44: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +46: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +24: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +44: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +11: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +11: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +55: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +55: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +48: [2023-04-29 09:45:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +25: [2023-04-29 09:45:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +29: [2023-04-29 09:45:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +63: [2023-04-29 09:45:13,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +63: [2023-04-29 09:45:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +46: [2023-04-29 09:45:13,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 2: [2023-04-29 09:45:13,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:13,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +63: [2023-04-29 09:45:13,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +17: [2023-04-29 09:45:13,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +10: [2023-04-29 09:45:13,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +47: [2023-04-29 09:45:13,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +48: [2023-04-29 09:45:13,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +24: [2023-04-29 09:45:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:13,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +22: [2023-04-29 09:45:13,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +47: [2023-04-29 09:45:13,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +24: [2023-04-29 09:45:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:13,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +24: [2023-04-29 09:45:13,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +63: [2023-04-29 09:45:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +24: [2023-04-29 09:45:13,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +61: [2023-04-29 09:45:13,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +21: [2023-04-29 09:45:13,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +21: [2023-04-29 09:45:13,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +21: [2023-04-29 09:45:13,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +55: [2023-04-29 09:45:13,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +55: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +22: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +55: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +10: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +55: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +22: [2023-04-29 09:45:13,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:13,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:13,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:13,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +42: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +42: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +11: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +42: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +42: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +13: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +17: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +47: [2023-04-29 09:45:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +60: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +17: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +22: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +10: [2023-04-29 09:45:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +33: [2023-04-29 09:45:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:13,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +10: [2023-04-29 09:45:13,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +21: [2023-04-29 09:45:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +60: [2023-04-29 09:45:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +42: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +17: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +17: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:13,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +42: [2023-04-29 09:45:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +17: [2023-04-29 09:45:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +54: [2023-04-29 09:45:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +54: [2023-04-29 09:45:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +60: [2023-04-29 09:45:13,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +10: [2023-04-29 09:45:13,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +42: [2023-04-29 09:45:13,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +22: [2023-04-29 09:45:13,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +33: [2023-04-29 09:45:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +22: [2023-04-29 09:45:13,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +33: [2023-04-29 09:45:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +61: [2023-04-29 09:45:13,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +60: [2023-04-29 09:45:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:13,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:13,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:13,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +60: [2023-04-29 09:45:13,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +61: [2023-04-29 09:45:13,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +61: [2023-04-29 09:45:13,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +57: [2023-04-29 09:45:13,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +61: [2023-04-29 09:45:13,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +34: [2023-04-29 09:45:13,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:13,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:13,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:13,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:13,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +54: [2023-04-29 09:45:13,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +22: [2023-04-29 09:45:13,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +48: [2023-04-29 09:45:13,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 8: [2023-04-29 09:45:13,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +61: [2023-04-29 09:45:13,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +56: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +54: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt... +57: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +29: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:13,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:13,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:13,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +48: [2023-04-29 09:45:13,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:13,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +55: [2023-04-29 09:45:13,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +10: [2023-04-29 09:45:13,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +55: [2023-04-29 09:45:13,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:13,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:13,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +10: [2023-04-29 09:45:13,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +11: [2023-04-29 09:45:13,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:13,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:13,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +56: [2023-04-29 09:45:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:13,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +23: [2023-04-29 09:45:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +34: [2023-04-29 09:45:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +22: [2023-04-29 09:45:13,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:13,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +23: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:13,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +23: [2023-04-29 09:45:13,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +21: [2023-04-29 09:45:13,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:13,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:13,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:13,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:13,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +13: [2023-04-29 09:45:13,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +56: [2023-04-29 09:45:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:13,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:13,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:13,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +63: [2023-04-29 09:45:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +11: [2023-04-29 09:45:13,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:13,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +63: [2023-04-29 09:45:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +30: [2023-04-29 09:45:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +13: [2023-04-29 09:45:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +23: [2023-04-29 09:45:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +16: [2023-04-29 09:45:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +16: [2023-04-29 09:45:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +13: [2023-04-29 09:45:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +53: [2023-04-29 09:45:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +53: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +53: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +34: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +63: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +63: [2023-04-29 09:45:13,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +63: [2023-04-29 09:45:13,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +10: [2023-04-29 09:45:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +57: [2023-04-29 09:45:13,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +30: [2023-04-29 09:45:13,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +57: [2023-04-29 09:45:13,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +18: [2023-04-29 09:45:13,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +22: [2023-04-29 09:45:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +61: [2023-04-29 09:45:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:13,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:13,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +56: [2023-04-29 09:45:13,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:13,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +56: [2023-04-29 09:45:13,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +37: [2023-04-29 09:45:13,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:13,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:13,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:13,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +37: [2023-04-29 09:45:13,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +28: [2023-04-29 09:45:13,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:13,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +63: [2023-04-29 09:45:13,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +28: [2023-04-29 09:45:13,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +61: [2023-04-29 09:45:13,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 3: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 3: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +61: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +25: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +30: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +18: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +15: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +57: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +37: [2023-04-29 09:45:13,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:13,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +30: [2023-04-29 09:45:13,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +18: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +25: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:13,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:13,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +37: [2023-04-29 09:45:13,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:13,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 8: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +38: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +38: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +38: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +38: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +30: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_00-model_states.pt. +46: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:13,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +10: [2023-04-29 09:45:13,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +56: [2023-04-29 09:45:13,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +28: [2023-04-29 09:45:13,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +28: [2023-04-29 09:45:13,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:13,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +10: [2023-04-29 09:45:13,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +62: [2023-04-29 09:45:13,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:13,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +62: [2023-04-29 09:45:13,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:13,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +34: [2023-04-29 09:45:13,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 6: [2023-04-29 09:45:13,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +34: [2023-04-29 09:45:13,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:13,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:13,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:13,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 6: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... + 6: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +18: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +62: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +46: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +23: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:13,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +46: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +38: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +18: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +38: [2023-04-29 09:45:14,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:14,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +63: [2023-04-29 09:45:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +30: [2023-04-29 09:45:14,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt... +46: [2023-04-29 09:45:14,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:14,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +23: [2023-04-29 09:45:14,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:14,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +56: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +18: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +18: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +42: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +62: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:14,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:14,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +18: [2023-04-29 09:45:14,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +18: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +54: [2023-04-29 09:45:14,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +53: [2023-04-29 09:45:14,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:14,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:14,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:14,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +47: [2023-04-29 09:45:14,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +16: [2023-04-29 09:45:14,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:14,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +47: [2023-04-29 09:45:14,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +53: [2023-04-29 09:45:14,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:14,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +42: [2023-04-29 09:45:14,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +42: [2023-04-29 09:45:14,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:14,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +60: [2023-04-29 09:45:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +25: [2023-04-29 09:45:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +54: [2023-04-29 09:45:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +54: [2023-04-29 09:45:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +54: [2023-04-29 09:45:14,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +15: [2023-04-29 09:45:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:14,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +17: [2023-04-29 09:45:14,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:14,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +54: [2023-04-29 09:45:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +13: [2023-04-29 09:45:14,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +13: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +15: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +60: [2023-04-29 09:45:14,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +19: [2023-04-29 09:45:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +52: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +29: [2023-04-29 09:45:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +55: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +53: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +32: [2023-04-29 09:45:14,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +39: [2023-04-29 09:45:14,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +54: [2023-04-29 09:45:14,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +39: [2023-04-29 09:45:14,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +54: [2023-04-29 09:45:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +52: [2023-04-29 09:45:14,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +39: [2023-04-29 09:45:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:14,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +63: [2023-04-29 09:45:14,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +54: [2023-04-29 09:45:14,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +39: [2023-04-29 09:45:14,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +55: [2023-04-29 09:45:14,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:14,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:14,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +17: [2023-04-29 09:45:14,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +42: [2023-04-29 09:45:14,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:14,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +45: [2023-04-29 09:45:14,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +53: [2023-04-29 09:45:14,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:14,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +39: [2023-04-29 09:45:14,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +45: [2023-04-29 09:45:14,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +58: [2023-04-29 09:45:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +58: [2023-04-29 09:45:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +58: [2023-04-29 09:45:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +24: [2023-04-29 09:45:14,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +63: [2023-04-29 09:45:14,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +24: [2023-04-29 09:45:14,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +24: [2023-04-29 09:45:14,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:14,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +60: [2023-04-29 09:45:14,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +60: [2023-04-29 09:45:14,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +25: [2023-04-29 09:45:14,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +13: [2023-04-29 09:45:14,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +42: [2023-04-29 09:45:14,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:14,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +54: [2023-04-29 09:45:14,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_00-model_states.pt. +29: [2023-04-29 09:45:14,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:14,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +19: [2023-04-29 09:45:14,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +17: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +19: [2023-04-29 09:45:14,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +60: [2023-04-29 09:45:14,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +19: [2023-04-29 09:45:14,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +42: [2023-04-29 09:45:14,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +17: [2023-04-29 09:45:14,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:14,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:14,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:14,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +17: [2023-04-29 09:45:14,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +62: [2023-04-29 09:45:14,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +41: [2023-04-29 09:45:14,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt... +42: [2023-04-29 09:45:14,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +42: [2023-04-29 09:45:14,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +42: [2023-04-29 09:45:14,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:14,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +60: [2023-04-29 09:45:14,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +61: [2023-04-29 09:45:14,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:14,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +42: [2023-04-29 09:45:14,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +55: [2023-04-29 09:45:14,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:14,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 4: [2023-04-29 09:45:14,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +13: [2023-04-29 09:45:14,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +41: [2023-04-29 09:45:14,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +41: [2023-04-29 09:45:14,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +41: [2023-04-29 09:45:14,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +41: [2023-04-29 09:45:14,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +23: [2023-04-29 09:45:14,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +45: [2023-04-29 09:45:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:14,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:14,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +20: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:14,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +20: [2023-04-29 09:45:14,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +45: [2023-04-29 09:45:14,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:14,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +55: [2023-04-29 09:45:14,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:14,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:14,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +13: [2023-04-29 09:45:14,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +38: [2023-04-29 09:45:14,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +28: [2023-04-29 09:45:14,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +45: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +45: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:14,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +46: [2023-04-29 09:45:14,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +23: [2023-04-29 09:45:14,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +62: [2023-04-29 09:45:14,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:14,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +20: [2023-04-29 09:45:14,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +13: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +20: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:14,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +62: [2023-04-29 09:45:14,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:14,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:14,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +38: [2023-04-29 09:45:14,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +46: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +31: [2023-04-29 09:45:14,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:14,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +31: [2023-04-29 09:45:14,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +62: [2023-04-29 09:45:14,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +30: [2023-04-29 09:45:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +30: [2023-04-29 09:45:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +30: [2023-04-29 09:45:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:14,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +30: [2023-04-29 09:45:14,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +47: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +28: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +16: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +31: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +23: [2023-04-29 09:45:14,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:14,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +28: [2023-04-29 09:45:14,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +31: [2023-04-29 09:45:14,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:14,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:14,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +26: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +58: [2023-04-29 09:45:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +61: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:14,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +58: [2023-04-29 09:45:14,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +61: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +58: [2023-04-29 09:45:14,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +29: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +32: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +18: [2023-04-29 09:45:14,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:14,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +41: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +53: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +23: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +53: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +30: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +53: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +30: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:14,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:14,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +16: [2023-04-29 09:45:14,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +41: [2023-04-29 09:45:14,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +53: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +23: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +26: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +26: [2023-04-29 09:45:14,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +43: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +26: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +43: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +42: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +19: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +42: [2023-04-29 09:45:14,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:14,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +24: [2023-04-29 09:45:14,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +24: [2023-04-29 09:45:14,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +24: [2023-04-29 09:45:14,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +24: [2023-04-29 09:45:14,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +23: [2023-04-29 09:45:14,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +53: [2023-04-29 09:45:14,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +39: [2023-04-29 09:45:14,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:14,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +32: [2023-04-29 09:45:14,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:14,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +37: [2023-04-29 09:45:14,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +61: [2023-04-29 09:45:14,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +21: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +21: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +38: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +38: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +53: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +52: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +21: [2023-04-29 09:45:14,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +21: [2023-04-29 09:45:14,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +61: [2023-04-29 09:45:14,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:14,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +62: [2023-04-29 09:45:14,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +16: [2023-04-29 09:45:14,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +54: [2023-04-29 09:45:14,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:14,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:14,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +46: [2023-04-29 09:45:14,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:14,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:14,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +32: [2023-04-29 09:45:14,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +38: [2023-04-29 09:45:14,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:14,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +38: [2023-04-29 09:45:14,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +46: [2023-04-29 09:45:14,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +46: [2023-04-29 09:45:14,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +23: [2023-04-29 09:45:14,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +32: [2023-04-29 09:45:14,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +47: [2023-04-29 09:45:14,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:14,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +12: [2023-04-29 09:45:14,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +36: [2023-04-29 09:45:14,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +36: [2023-04-29 09:45:14,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +36: [2023-04-29 09:45:14,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +36: [2023-04-29 09:45:14,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +24: [2023-04-29 09:45:14,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +62: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +43: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +43: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +43: [2023-04-29 09:45:14,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +43: [2023-04-29 09:45:14,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +41: [2023-04-29 09:45:14,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +12: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +52: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +33: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +33: [2023-04-29 09:45:14,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +33: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +36: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +39: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +19: [2023-04-29 09:45:14,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +39: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +59: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +59: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +45: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +45: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +38: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +38: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +38: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +38: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +34: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +34: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +34: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +45: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +45: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +26: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +12: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +59: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +21: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +41: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +49: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +59: [2023-04-29 09:45:14,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +49: [2023-04-29 09:45:14,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:14,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:14,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +21: [2023-04-29 09:45:14,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +39: [2023-04-29 09:45:14,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +27: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +52: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +27: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +45: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +27: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +54: [2023-04-29 09:45:14,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +21: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +54: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +15: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +15: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +15: [2023-04-29 09:45:14,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +49: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +45: [2023-04-29 09:45:14,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +32: [2023-04-29 09:45:14,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:14,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +49: [2023-04-29 09:45:14,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +49: [2023-04-29 09:45:14,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +32: [2023-04-29 09:45:14,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +32: [2023-04-29 09:45:14,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +20: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +27: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +52: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +58: [2023-04-29 09:45:14,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +49: [2023-04-29 09:45:14,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +58: [2023-04-29 09:45:14,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +41: [2023-04-29 09:45:14,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +54: [2023-04-29 09:45:14,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +54: [2023-04-29 09:45:14,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +41: [2023-04-29 09:45:14,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +32: [2023-04-29 09:45:14,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +32: [2023-04-29 09:45:14,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +58: [2023-04-29 09:45:14,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +58: [2023-04-29 09:45:14,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +19: [2023-04-29 09:45:14,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +12: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +12: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +20: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +39: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +58: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +25: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +58: [2023-04-29 09:45:14,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +43: [2023-04-29 09:45:14,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +58: [2023-04-29 09:45:14,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +25: [2023-04-29 09:45:14,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +58: [2023-04-29 09:45:14,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +25: [2023-04-29 09:45:14,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +59: [2023-04-29 09:45:14,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +58: [2023-04-29 09:45:14,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +42: [2023-04-29 09:45:14,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:14,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +34: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:14,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +38: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +27: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +58: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:14,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:14,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +58: [2023-04-29 09:45:14,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:14,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +58: [2023-04-29 09:45:14,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +19: [2023-04-29 09:45:14,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +19: [2023-04-29 09:45:14,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:14,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:14,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:14,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +43: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +33: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +17: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +17: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +31: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +39: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +39: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +45: [2023-04-29 09:45:14,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +20: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +30: [2023-04-29 09:45:14,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +43: [2023-04-29 09:45:14,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:14,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:14,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +15: [2023-04-29 09:45:14,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +42: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +15: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +40: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +20: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +32: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +51: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +51: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +59: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +19: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +19: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +27: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +15: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:14,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +27: [2023-04-29 09:45:14,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +59: [2023-04-29 09:45:14,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:14,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +45: [2023-04-29 09:45:14,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +19: [2023-04-29 09:45:14,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +51: [2023-04-29 09:45:14,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:14,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +51: [2023-04-29 09:45:14,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +51: [2023-04-29 09:45:14,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +40: [2023-04-29 09:45:14,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +41: [2023-04-29 09:45:14,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:14,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +30: [2023-04-29 09:45:14,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +40: [2023-04-29 09:45:14,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +51: [2023-04-29 09:45:14,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:14,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:14,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +42: [2023-04-29 09:45:14,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +29: [2023-04-29 09:45:14,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +29: [2023-04-29 09:45:14,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +51: [2023-04-29 09:45:14,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +31: [2023-04-29 09:45:14,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:14,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +16: [2023-04-29 09:45:14,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +51: [2023-04-29 09:45:14,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +29: [2023-04-29 09:45:14,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +43: [2023-04-29 09:45:14,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:14,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +41: [2023-04-29 09:45:14,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:14,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +12: [2023-04-29 09:45:14,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:14,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +31: [2023-04-29 09:45:14,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +51: [2023-04-29 09:45:14,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:14,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 9: [2023-04-29 09:45:14,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:14,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +27: [2023-04-29 09:45:14,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:14,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:14,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +31: [2023-04-29 09:45:14,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:14,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +20: [2023-04-29 09:45:14,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +30: [2023-04-29 09:45:14,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +25: [2023-04-29 09:45:14,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:14,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +26: [2023-04-29 09:45:14,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +50: [2023-04-29 09:45:14,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:14,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:14,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +43: [2023-04-29 09:45:14,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +30: [2023-04-29 09:45:14,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +24: [2023-04-29 09:45:14,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +26: [2023-04-29 09:45:14,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:14,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +36: [2023-04-29 09:45:14,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +21: [2023-04-29 09:45:14,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +28: [2023-04-29 09:45:14,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +28: [2023-04-29 09:45:14,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +35: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +31: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +35: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +35: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +28: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +50: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +28: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +50: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:14,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +50: [2023-04-29 09:45:14,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:14,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +50: [2023-04-29 09:45:14,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +24: [2023-04-29 09:45:14,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +43: [2023-04-29 09:45:14,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:14,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:14,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +26: [2023-04-29 09:45:14,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +26: [2023-04-29 09:45:14,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +36: [2023-04-29 09:45:14,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +17: [2023-04-29 09:45:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +17: [2023-04-29 09:45:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +16: [2023-04-29 09:45:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +36: [2023-04-29 09:45:14,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +27: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +20: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +41: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +48: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +48: [2023-04-29 09:45:14,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +30: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +49: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +26: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +36: [2023-04-29 09:45:14,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +29: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +29: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +43: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +12: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +43: [2023-04-29 09:45:14,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:14,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +51: [2023-04-29 09:45:14,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +16: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +29: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +20: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +19: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +45: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:14,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +26: [2023-04-29 09:45:14,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +21: [2023-04-29 09:45:14,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:14,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:14,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +37: [2023-04-29 09:45:14,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +41: [2023-04-29 09:45:14,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +37: [2023-04-29 09:45:14,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +37: [2023-04-29 09:45:14,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +19: [2023-04-29 09:45:14,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +12: [2023-04-29 09:45:14,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +24: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +31: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +41: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +11: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +11: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +51: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +21: [2023-04-29 09:45:14,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:14,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:14,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +45: [2023-04-29 09:45:14,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +24: [2023-04-29 09:45:14,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +30: [2023-04-29 09:45:14,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +21: [2023-04-29 09:45:14,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +19: [2023-04-29 09:45:14,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +21: [2023-04-29 09:45:14,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +41: [2023-04-29 09:45:14,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +35: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +50: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +11: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +11: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +50: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +35: [2023-04-29 09:45:14,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +49: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +35: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +39: [2023-04-29 09:45:14,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +19: [2023-04-29 09:45:14,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +51: [2023-04-29 09:45:14,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +35: [2023-04-29 09:45:14,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +38: [2023-04-29 09:45:14,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +41: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 1: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +21: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +27: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +18: [2023-04-29 09:45:14,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +18: [2023-04-29 09:45:14,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +39: [2023-04-29 09:45:14,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 1: [2023-04-29 09:45:14,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +45: [2023-04-29 09:45:14,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:14,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +35: [2023-04-29 09:45:14,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +35: [2023-04-29 09:45:14,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:14,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:14,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +34: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +12: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +35: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +14: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +38: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +58: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +15: [2023-04-29 09:45:14,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +38: [2023-04-29 09:45:14,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:14,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +49: [2023-04-29 09:45:14,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +49: [2023-04-29 09:45:14,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 4: [2023-04-29 09:45:14,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +49: [2023-04-29 09:45:14,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +38: [2023-04-29 09:45:14,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +14: [2023-04-29 09:45:14,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +20: [2023-04-29 09:45:14,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +33: [2023-04-29 09:45:14,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:14,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +16: [2023-04-29 09:45:14,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +35: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +28: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +43: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +58: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +58: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +15: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +34: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +58: [2023-04-29 09:45:14,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +34: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +38: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +16: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +49: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +34: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +27: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +28: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:14,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:14,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:14,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:14,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +30: [2023-04-29 09:45:14,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +45: [2023-04-29 09:45:14,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +34: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +30: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +30: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_18-model_01-model_states.pt. +28: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +36: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +15: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +15: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +37: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +32: [2023-04-29 09:45:14,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +33: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +43: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +36: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +59: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +59: [2023-04-29 09:45:14,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +20: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +38: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +37: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:14,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +39: [2023-04-29 09:45:14,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +39: [2023-04-29 09:45:14,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +33: [2023-04-29 09:45:14,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:14,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +12: [2023-04-29 09:45:14,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +59: [2023-04-29 09:45:14,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +20: [2023-04-29 09:45:14,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +48: [2023-04-29 09:45:14,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:14,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +15: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +12: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +48: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +14: [2023-04-29 09:45:14,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +37: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +14: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +44: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +44: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +59: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +44: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:14,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +39: [2023-04-29 09:45:14,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +44: [2023-04-29 09:45:14,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +14: [2023-04-29 09:45:14,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +11: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +59: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +43: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +14: [2023-04-29 09:45:14,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +45: [2023-04-29 09:45:14,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +60: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +60: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +46: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +46: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +50: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +40: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +46: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +25: [2023-04-29 09:45:14,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +59: [2023-04-29 09:45:14,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +11: [2023-04-29 09:45:14,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +49: [2023-04-29 09:45:14,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +59: [2023-04-29 09:45:14,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +44: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +14: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +44: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +60: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +59: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +59: [2023-04-29 09:45:14,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +44: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +44: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +44: [2023-04-29 09:45:14,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +60: [2023-04-29 09:45:14,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +44: [2023-04-29 09:45:14,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +43: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +27: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +44: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +59: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +27: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +25: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +22: [2023-04-29 09:45:14,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +34: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +25: [2023-04-29 09:45:14,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +34: [2023-04-29 09:45:14,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +49: [2023-04-29 09:45:14,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:14,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +58: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +57: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +57: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +12: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +59: [2023-04-29 09:45:14,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +57: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +49: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +50: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +49: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +49: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +18: [2023-04-29 09:45:14,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +22: [2023-04-29 09:45:14,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +22: [2023-04-29 09:45:14,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +51: [2023-04-29 09:45:14,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +58: [2023-04-29 09:45:14,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:14,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +17: [2023-04-29 09:45:14,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +51: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +51: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +34: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +34: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +49: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +50: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 0: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +27: [2023-04-29 09:45:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +58: [2023-04-29 09:45:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +12: [2023-04-29 09:45:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +43: [2023-04-29 09:45:14,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +33: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +33: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +58: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +10: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +25: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +10: [2023-04-29 09:45:14,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +10: [2023-04-29 09:45:14,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +27: [2023-04-29 09:45:14,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +40: [2023-04-29 09:45:14,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +30: [2023-04-29 09:45:14,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +14: [2023-04-29 09:45:14,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:14,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 9: [2023-04-29 09:45:14,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +47: [2023-04-29 09:45:14,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +47: [2023-04-29 09:45:14,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +47: [2023-04-29 09:45:14,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +43: [2023-04-29 09:45:14,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +12: [2023-04-29 09:45:14,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +33: [2023-04-29 09:45:14,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +33: [2023-04-29 09:45:14,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +30: [2023-04-29 09:45:14,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +30: [2023-04-29 09:45:14,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +25: [2023-04-29 09:45:14,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +33: [2023-04-29 09:45:14,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +33: [2023-04-29 09:45:14,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +12: [2023-04-29 09:45:14,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +45: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +12: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +14: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:14,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:14,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:14,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +33: [2023-04-29 09:45:14,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +30: [2023-04-29 09:45:14,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt... +33: [2023-04-29 09:45:14,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +12: [2023-04-29 09:45:14,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +51: [2023-04-29 09:45:14,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +12: [2023-04-29 09:45:14,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +43: [2023-04-29 09:45:14,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +25: [2023-04-29 09:45:14,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:14,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +17: [2023-04-29 09:45:14,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +29: [2023-04-29 09:45:14,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:14,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +29: [2023-04-29 09:45:14,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +60: [2023-04-29 09:45:14,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:14,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +46: [2023-04-29 09:45:14,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:14,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:14,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +46: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +44: [2023-04-29 09:45:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:14,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +63: [2023-04-29 09:45:14,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +63: [2023-04-29 09:45:14,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +63: [2023-04-29 09:45:14,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +31: [2023-04-29 09:45:14,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +44: [2023-04-29 09:45:14,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +44: [2023-04-29 09:45:14,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +29: [2023-04-29 09:45:14,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +29: [2023-04-29 09:45:14,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +29: [2023-04-29 09:45:14,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +17: [2023-04-29 09:45:14,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +29: [2023-04-29 09:45:14,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +60: [2023-04-29 09:45:14,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:14,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:14,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +27: [2023-04-29 09:45:14,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +14: [2023-04-29 09:45:14,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +57: [2023-04-29 09:45:14,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:14,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +31: [2023-04-29 09:45:14,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +27: [2023-04-29 09:45:14,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +10: [2023-04-29 09:45:14,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +57: [2023-04-29 09:45:14,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +20: [2023-04-29 09:45:14,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +57: [2023-04-29 09:45:14,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +31: [2023-04-29 09:45:14,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +51: [2023-04-29 09:45:14,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +51: [2023-04-29 09:45:14,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:14,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +14: [2023-04-29 09:45:14,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +35: [2023-04-29 09:45:14,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +31: [2023-04-29 09:45:14,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +10: [2023-04-29 09:45:14,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +31: [2023-04-29 09:45:14,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +14: [2023-04-29 09:45:14,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +47: [2023-04-29 09:45:14,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +20: [2023-04-29 09:45:14,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +31: [2023-04-29 09:45:14,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +14: [2023-04-29 09:45:14,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +56: [2023-04-29 09:45:14,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +27: [2023-04-29 09:45:14,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +56: [2023-04-29 09:45:14,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +10: [2023-04-29 09:45:14,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +56: [2023-04-29 09:45:14,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:14,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +47: [2023-04-29 09:45:14,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +27: [2023-04-29 09:45:14,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +35: [2023-04-29 09:45:14,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:14,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:14,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +50: [2023-04-29 09:45:14,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +50: [2023-04-29 09:45:14,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:14,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:14,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +55: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +55: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +55: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +50: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +56: [2023-04-29 09:45:14,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +10: [2023-04-29 09:45:14,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +35: [2023-04-29 09:45:14,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +35: [2023-04-29 09:45:14,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +56: [2023-04-29 09:45:14,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:14,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:14,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +51: [2023-04-29 09:45:14,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:14,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +37: [2023-04-29 09:45:14,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:14,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:14,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:14,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:14,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +35: [2023-04-29 09:45:14,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:14,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:14,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +51: [2023-04-29 09:45:14,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +10: [2023-04-29 09:45:14,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:14,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:14,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +28: [2023-04-29 09:45:14,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +35: [2023-04-29 09:45:14,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:14,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +20: [2023-04-29 09:45:14,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +37: [2023-04-29 09:45:14,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:14,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:14,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +51: [2023-04-29 09:45:14,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +57: [2023-04-29 09:45:14,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +57: [2023-04-29 09:45:14,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:14,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +57: [2023-04-29 09:45:14,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:14,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:14,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +56: [2023-04-29 09:45:14,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +10: [2023-04-29 09:45:14,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +48: [2023-04-29 09:45:14,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:14,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:14,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +59: [2023-04-29 09:45:14,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +35: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +56: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +10: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +10: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +20: [2023-04-29 09:45:14,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +37: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +28: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +28: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +28: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +13: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +13: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +13: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +59: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:14,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +48: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +21: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +62: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +62: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +50: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +48: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +28: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +51: [2023-04-29 09:45:14,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +21: [2023-04-29 09:45:14,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:14,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +35: [2023-04-29 09:45:14,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +37: [2023-04-29 09:45:14,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +62: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +53: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +53: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +53: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +53: [2023-04-29 09:45:14,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +20: [2023-04-29 09:45:14,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +59: [2023-04-29 09:45:14,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +34: [2023-04-29 09:45:14,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:14,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:14,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +21: [2023-04-29 09:45:14,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +48: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +48: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +18: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +21: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +50: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +21: [2023-04-29 09:45:14,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +33: [2023-04-29 09:45:14,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:14,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +49: [2023-04-29 09:45:14,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +44: [2023-04-29 09:45:14,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:14,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:14,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +53: [2023-04-29 09:45:14,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:14,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:14,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:14,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +52: [2023-04-29 09:45:14,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +52: [2023-04-29 09:45:14,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +18: [2023-04-29 09:45:14,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +18: [2023-04-29 09:45:14,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +52: [2023-04-29 09:45:14,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +48: [2023-04-29 09:45:14,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +49: [2023-04-29 09:45:14,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +18: [2023-04-29 09:45:14,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +22: [2023-04-29 09:45:14,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +34: [2023-04-29 09:45:14,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +49: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +20: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +44: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +44: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +22: [2023-04-29 09:45:14,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +49: [2023-04-29 09:45:14,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:14,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:14,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +55: [2023-04-29 09:45:14,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +54: [2023-04-29 09:45:14,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +54: [2023-04-29 09:45:14,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +54: [2023-04-29 09:45:14,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +34: [2023-04-29 09:45:14,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:14,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:14,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +54: [2023-04-29 09:45:14,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +33: [2023-04-29 09:45:14,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +18: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +18: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +49: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +50: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +50: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:14,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +33: [2023-04-29 09:45:14,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +42: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +49: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +14: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +44: [2023-04-29 09:45:14,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +42: [2023-04-29 09:45:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +56: [2023-04-29 09:45:14,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:14,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +60: [2023-04-29 09:45:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +30: [2023-04-29 09:45:14,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +44: [2023-04-29 09:45:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +14: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +35: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +42: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:14,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +59: [2023-04-29 09:45:14,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:14,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +30: [2023-04-29 09:45:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_00-model_states.pt. +63: [2023-04-29 09:45:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +50: [2023-04-29 09:45:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +11: [2023-04-29 09:45:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +11: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +60: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +56: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +56: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +11: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +11: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +42: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +11: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +34: [2023-04-29 09:45:14,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:14,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +60: [2023-04-29 09:45:14,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +22: [2023-04-29 09:45:14,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +62: [2023-04-29 09:45:14,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:14,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +62: [2023-04-29 09:45:14,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +63: [2023-04-29 09:45:14,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +30: [2023-04-29 09:45:14,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt... +60: [2023-04-29 09:45:14,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +61: [2023-04-29 09:45:14,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +61: [2023-04-29 09:45:14,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +36: [2023-04-29 09:45:14,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +36: [2023-04-29 09:45:14,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +61: [2023-04-29 09:45:14,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +61: [2023-04-29 09:45:14,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +62: [2023-04-29 09:45:14,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +11: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +50: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +44: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +63: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +11: [2023-04-29 09:45:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +11: [2023-04-29 09:45:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +11: [2023-04-29 09:45:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +62: [2023-04-29 09:45:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +11: [2023-04-29 09:45:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +44: [2023-04-29 09:45:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:14,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:14,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +59: [2023-04-29 09:45:14,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +59: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +36: [2023-04-29 09:45:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +10: [2023-04-29 09:45:14,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +60: [2023-04-29 09:45:14,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:14,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +57: [2023-04-29 09:45:14,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +57: [2023-04-29 09:45:14,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +33: [2023-04-29 09:45:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +57: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +52: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +34: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +22: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 2: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 2: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 2: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +34: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +52: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +34: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +14: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +44: [2023-04-29 09:45:14,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:14,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +46: [2023-04-29 09:45:14,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +61: [2023-04-29 09:45:14,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:14,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:14,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +57: [2023-04-29 09:45:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +10: [2023-04-29 09:45:14,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +57: [2023-04-29 09:45:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +47: [2023-04-29 09:45:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +44: [2023-04-29 09:45:14,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +22: [2023-04-29 09:45:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +47: [2023-04-29 09:45:14,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +50: [2023-04-29 09:45:14,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +60: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +54: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +38: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +38: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +38: [2023-04-29 09:45:14,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +14: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +14: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +10: [2023-04-29 09:45:14,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +47: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +47: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:14,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +44: [2023-04-29 09:45:14,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +62: [2023-04-29 09:45:14,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:14,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +62: [2023-04-29 09:45:14,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:14,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +14: [2023-04-29 09:45:14,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +40: [2023-04-29 09:45:14,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +38: [2023-04-29 09:45:14,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +38: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +47: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +50: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +40: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_40-model_01-model_states.pt. +38: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +47: [2023-04-29 09:45:14,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +38: [2023-04-29 09:45:14,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:14,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:14,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +61: [2023-04-29 09:45:14,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:14,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:14,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:14,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +61: [2023-04-29 09:45:14,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +46: [2023-04-29 09:45:14,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:14,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +14: [2023-04-29 09:45:14,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +63: [2023-04-29 09:45:14,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +61: [2023-04-29 09:45:14,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +23: [2023-04-29 09:45:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +23: [2023-04-29 09:45:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +23: [2023-04-29 09:45:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +23: [2023-04-29 09:45:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +36: [2023-04-29 09:45:14,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +63: [2023-04-29 09:45:14,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:14,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +46: [2023-04-29 09:45:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +63: [2023-04-29 09:45:14,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +36: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +36: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +32: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +63: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +10: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +21: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +56: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +48: [2023-04-29 09:45:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +63: [2023-04-29 09:45:14,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +16: [2023-04-29 09:45:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +16: [2023-04-29 09:45:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +16: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +55: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +53: [2023-04-29 09:45:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +53: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +63: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +56: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +13: [2023-04-29 09:45:14,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +53: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +37: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +13: [2023-04-29 09:45:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +10: [2023-04-29 09:45:14,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +55: [2023-04-29 09:45:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +21: [2023-04-29 09:45:14,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +40: [2023-04-29 09:45:14,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +42: [2023-04-29 09:45:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:14,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +37: [2023-04-29 09:45:14,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +37: [2023-04-29 09:45:14,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:14,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +10: [2023-04-29 09:45:14,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:14,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +40: [2023-04-29 09:45:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +21: [2023-04-29 09:45:14,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +55: [2023-04-29 09:45:14,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:14,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +55: [2023-04-29 09:45:14,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:14,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:14,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt... +47: [2023-04-29 09:45:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:14,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +22: [2023-04-29 09:45:14,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +22: [2023-04-29 09:45:14,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +42: [2023-04-29 09:45:14,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +22: [2023-04-29 09:45:14,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +56: [2023-04-29 09:45:14,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:14,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +10: [2023-04-29 09:45:14,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +47: [2023-04-29 09:45:14,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:14,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +55: [2023-04-29 09:45:14,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +55: [2023-04-29 09:45:14,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +47: [2023-04-29 09:45:14,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +13: [2023-04-29 09:45:14,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +11: [2023-04-29 09:45:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +57: [2023-04-29 09:45:14,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +23: [2023-04-29 09:45:14,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +62: [2023-04-29 09:45:14,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:14,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +56: [2023-04-29 09:45:14,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +42: [2023-04-29 09:45:14,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:14,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:14,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +60: [2023-04-29 09:45:14,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:14,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +48: [2023-04-29 09:45:14,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:14,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +53: [2023-04-29 09:45:14,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:14,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +60: [2023-04-29 09:45:14,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:14,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +52: [2023-04-29 09:45:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:14,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:14,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:14,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:14,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:14,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +11: [2023-04-29 09:45:14,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +11: [2023-04-29 09:45:14,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +52: [2023-04-29 09:45:14,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +16: [2023-04-29 09:45:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +53: [2023-04-29 09:45:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:14,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:14,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +13: [2023-04-29 09:45:14,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +11: [2023-04-29 09:45:14,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +42: [2023-04-29 09:45:14,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:14,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:14,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +56: [2023-04-29 09:45:14,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +54: [2023-04-29 09:45:14,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +52: [2023-04-29 09:45:14,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:14,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +54: [2023-04-29 09:45:14,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +52: [2023-04-29 09:45:14,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +21: [2023-04-29 09:45:14,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +61: [2023-04-29 09:45:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:14,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +56: [2023-04-29 09:45:14,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:14,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +10: [2023-04-29 09:45:14,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +42: [2023-04-29 09:45:14,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:14,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:14,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +22: [2023-04-29 09:45:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +61: [2023-04-29 09:45:14,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +22: [2023-04-29 09:45:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +22: [2023-04-29 09:45:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +10: [2023-04-29 09:45:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +56: [2023-04-29 09:45:14,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +62: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +54: [2023-04-29 09:45:14,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +54: [2023-04-29 09:45:14,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +54: [2023-04-29 09:45:14,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +22: [2023-04-29 09:45:14,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +61: [2023-04-29 09:45:14,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +54: [2023-04-29 09:45:14,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +56: [2023-04-29 09:45:14,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +52: [2023-04-29 09:45:14,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +38: [2023-04-29 09:45:14,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:14,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:14,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:14,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +60: [2023-04-29 09:45:14,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:14,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +38: [2023-04-29 09:45:14,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:14,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:14,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +61: [2023-04-29 09:45:14,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +54: [2023-04-29 09:45:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +62: [2023-04-29 09:45:14,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +54: [2023-04-29 09:45:14,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +46: [2023-04-29 09:45:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +52: [2023-04-29 09:45:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +62: [2023-04-29 09:45:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +61: [2023-04-29 09:45:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +60: [2023-04-29 09:45:14,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +11: [2023-04-29 09:45:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +63: [2023-04-29 09:45:14,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:14,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +54: [2023-04-29 09:45:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +36: [2023-04-29 09:45:14,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:14,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +46: [2023-04-29 09:45:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +11: [2023-04-29 09:45:14,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +63: [2023-04-29 09:45:14,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:14,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +62: [2023-04-29 09:45:14,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +36: [2023-04-29 09:45:14,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +40: [2023-04-29 09:45:14,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +36: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +62: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:14,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:14,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +36: [2023-04-29 09:45:14,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +61: [2023-04-29 09:45:14,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +11: [2023-04-29 09:45:14,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +61: [2023-04-29 09:45:14,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +32: [2023-04-29 09:45:14,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +61: [2023-04-29 09:45:14,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +13: [2023-04-29 09:45:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +38: [2023-04-29 09:45:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +36: [2023-04-29 09:45:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +36: [2023-04-29 09:45:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +13: [2023-04-29 09:45:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:14,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:14,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +25: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +61: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +25: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +36: [2023-04-29 09:45:14,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +40: [2023-04-29 09:45:14,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_00-model_states.pt. +38: [2023-04-29 09:45:14,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:14,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:14,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:14,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:14,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +25: [2023-04-29 09:45:14,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:14,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +40: [2023-04-29 09:45:14,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +38: [2023-04-29 09:45:14,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:14,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:14,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +13: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +25: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +46: [2023-04-29 09:45:14,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +38: [2023-04-29 09:45:14,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +41: [2023-04-29 09:45:14,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +41: [2023-04-29 09:45:14,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +41: [2023-04-29 09:45:14,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:14,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt... +25: [2023-04-29 09:45:14,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:14,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +47: [2023-04-29 09:45:14,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:14,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +18: [2023-04-29 09:45:14,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +61: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +24: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:14,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +24: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +13: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +52: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +16: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +22: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +62: [2023-04-29 09:45:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +62: [2023-04-29 09:45:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:14,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +61: [2023-04-29 09:45:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +24: [2023-04-29 09:45:14,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +24: [2023-04-29 09:45:14,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +23: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +52: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +52: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +37: [2023-04-29 09:45:14,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +19: [2023-04-29 09:45:14,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +47: [2023-04-29 09:45:14,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:14,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +16: [2023-04-29 09:45:14,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +55: [2023-04-29 09:45:14,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +47: [2023-04-29 09:45:14,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +13: [2023-04-29 09:45:14,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +37: [2023-04-29 09:45:14,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:14,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:14,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +23: [2023-04-29 09:45:14,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +47: [2023-04-29 09:45:14,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +23: [2023-04-29 09:45:14,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +55: [2023-04-29 09:45:14,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:14,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:14,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +23: [2023-04-29 09:45:14,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +13: [2023-04-29 09:45:14,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +23: [2023-04-29 09:45:14,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +23: [2023-04-29 09:45:14,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +23: [2023-04-29 09:45:14,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +19: [2023-04-29 09:45:14,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +22: [2023-04-29 09:45:14,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +54: [2023-04-29 09:45:14,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +22: [2023-04-29 09:45:14,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +54: [2023-04-29 09:45:14,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:14,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +15: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +45: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +15: [2023-04-29 09:45:14,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +45: [2023-04-29 09:45:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:14,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +15: [2023-04-29 09:45:14,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +45: [2023-04-29 09:45:14,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +25: [2023-04-29 09:45:14,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +41: [2023-04-29 09:45:14,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +19: [2023-04-29 09:45:14,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +19: [2023-04-29 09:45:14,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +25: [2023-04-29 09:45:14,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +26: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +41: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +41: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +26: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:14,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:14,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +17: [2023-04-29 09:45:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:14,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +54: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +28: [2023-04-29 09:45:14,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:14,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +17: [2023-04-29 09:45:14,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +14: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +17: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +43: [2023-04-29 09:45:14,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:14,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:14,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +14: [2023-04-29 09:45:14,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +14: [2023-04-29 09:45:14,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +41: [2023-04-29 09:45:14,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:14,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +24: [2023-04-29 09:45:14,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:14,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +24: [2023-04-29 09:45:14,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +21: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +24: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +21: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +29: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +24: [2023-04-29 09:45:14,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +39: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +29: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +19: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +23: [2023-04-29 09:45:14,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +15: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +15: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +41: [2023-04-29 09:45:14,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:14,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:14,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +19: [2023-04-29 09:45:14,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:14,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:14,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +26: [2023-04-29 09:45:14,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +26: [2023-04-29 09:45:14,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +26: [2023-04-29 09:45:14,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +19: [2023-04-29 09:45:14,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +58: [2023-04-29 09:45:14,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +24: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +41: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +58: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +58: [2023-04-29 09:45:14,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +58: [2023-04-29 09:45:14,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +49: [2023-04-29 09:45:14,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +49: [2023-04-29 09:45:14,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +49: [2023-04-29 09:45:14,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +49: [2023-04-29 09:45:14,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +45: [2023-04-29 09:45:14,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +45: [2023-04-29 09:45:14,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +31: [2023-04-29 09:45:14,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +49: [2023-04-29 09:45:14,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:14,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +49: [2023-04-29 09:45:14,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +24: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +21: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +49: [2023-04-29 09:45:14,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +23: [2023-04-29 09:45:14,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +24: [2023-04-29 09:45:14,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +15: [2023-04-29 09:45:14,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:14,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +23: [2023-04-29 09:45:14,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +29: [2023-04-29 09:45:14,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +29: [2023-04-29 09:45:14,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +29: [2023-04-29 09:45:14,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +40: [2023-04-29 09:45:14,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:14,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:14,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +40: [2023-04-29 09:45:14,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:14,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +40: [2023-04-29 09:45:14,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +12: [2023-04-29 09:45:14,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +29: [2023-04-29 09:45:14,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +26: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +26: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +26: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +51: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +51: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:14,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +40: [2023-04-29 09:45:14,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:14,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:14,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +40: [2023-04-29 09:45:14,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:14,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:14,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +51: [2023-04-29 09:45:14,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +51: [2023-04-29 09:45:14,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +43: [2023-04-29 09:45:14,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +17: [2023-04-29 09:45:14,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +51: [2023-04-29 09:45:14,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:14,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +51: [2023-04-29 09:45:14,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +45: [2023-04-29 09:45:14,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +31: [2023-04-29 09:45:14,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +51: [2023-04-29 09:45:14,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +51: [2023-04-29 09:45:14,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +31: [2023-04-29 09:45:14,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +20: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 4: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +25: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 2: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 2: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +27: [2023-04-29 09:45:14,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +28: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +20: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +26: [2023-04-29 09:45:14,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +39: [2023-04-29 09:45:14,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +20: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +20: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:14,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:14,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +29: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +39: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +27: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +15: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +29: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +30: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +30: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +29: [2023-04-29 09:45:14,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +17: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +20: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +15: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +20: [2023-04-29 09:45:14,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +20: [2023-04-29 09:45:14,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:14,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +18: [2023-04-29 09:45:14,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +18: [2023-04-29 09:45:14,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +18: [2023-04-29 09:45:14,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +26: [2023-04-29 09:45:14,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +12: [2023-04-29 09:45:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +45: [2023-04-29 09:45:14,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +19: [2023-04-29 09:45:14,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +20: [2023-04-29 09:45:14,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +20: [2023-04-29 09:45:14,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +58: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +12: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +43: [2023-04-29 09:45:14,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +43: [2023-04-29 09:45:14,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +19: [2023-04-29 09:45:14,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +17: [2023-04-29 09:45:14,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:14,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +11: [2023-04-29 09:45:14,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +11: [2023-04-29 09:45:14,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 4: [2023-04-29 09:45:14,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +58: [2023-04-29 09:45:14,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:14,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +11: [2023-04-29 09:45:14,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +21: [2023-04-29 09:45:14,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +49: [2023-04-29 09:45:14,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 4: [2023-04-29 09:45:14,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +11: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +26: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:14,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +21: [2023-04-29 09:45:14,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:14,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:14,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +25: [2023-04-29 09:45:14,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +43: [2023-04-29 09:45:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +29: [2023-04-29 09:45:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:14,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +53: [2023-04-29 09:45:14,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +29: [2023-04-29 09:45:14,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +53: [2023-04-29 09:45:14,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:14,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:14,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +53: [2023-04-29 09:45:14,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +16: [2023-04-29 09:45:14,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:14,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +14: [2023-04-29 09:45:14,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +28: [2023-04-29 09:45:14,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:14,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +27: [2023-04-29 09:45:14,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +39: [2023-04-29 09:45:14,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +49: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +21: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +53: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:14,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +53: [2023-04-29 09:45:14,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:14,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:14,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +29: [2023-04-29 09:45:14,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +17: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +21: [2023-04-29 09:45:14,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +58: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +26: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:14,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +26: [2023-04-29 09:45:14,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +21: [2023-04-29 09:45:14,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +17: [2023-04-29 09:45:14,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +18: [2023-04-29 09:45:14,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +20: [2023-04-29 09:45:14,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:14,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:14,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:14,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:14,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:14,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +18: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +18: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +58: [2023-04-29 09:45:14,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +11: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +11: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:14,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +15: [2023-04-29 09:45:14,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +15: [2023-04-29 09:45:14,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +48: [2023-04-29 09:45:14,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +21: [2023-04-29 09:45:14,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +22: [2023-04-29 09:45:14,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +22: [2023-04-29 09:45:14,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +11: [2023-04-29 09:45:14,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:14,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:14,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:14,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +34: [2023-04-29 09:45:14,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +11: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +28: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +28: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +48: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +48: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +28: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +26: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +19: [2023-04-29 09:45:14,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +51: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +51: [2023-04-29 09:45:14,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +20: [2023-04-29 09:45:14,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +22: [2023-04-29 09:45:14,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +13: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +22: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +51: [2023-04-29 09:45:14,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +34: [2023-04-29 09:45:14,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:14,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +51: [2023-04-29 09:45:14,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +34: [2023-04-29 09:45:14,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 4: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +15: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +15: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +15: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +40: [2023-04-29 09:45:14,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:14,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:14,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +26: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +26: [2023-04-29 09:45:14,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +26: [2023-04-29 09:45:14,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +13: [2023-04-29 09:45:14,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +19: [2023-04-29 09:45:14,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +19: [2023-04-29 09:45:14,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 9: [2023-04-29 09:45:14,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +42: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +26: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +42: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_02_optim_states.pt... +42: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_02_optim_states.pt... +45: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_02_optim_states.pt... +45: [2023-04-29 09:45:14,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_02_optim_states.pt... +49: [2023-04-29 09:45:14,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +49: [2023-04-29 09:45:14,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +49: [2023-04-29 09:45:14,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +29: [2023-04-29 09:45:14,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +24: [2023-04-29 09:45:14,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +24: [2023-04-29 09:45:14,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +24: [2023-04-29 09:45:14,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +24: [2023-04-29 09:45:14,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +26: [2023-04-29 09:45:14,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +17: [2023-04-29 09:45:14,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +10: [2023-04-29 09:45:14,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +10: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +10: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +42: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +42: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +29: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:14,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 9: [2023-04-29 09:45:14,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +27: [2023-04-29 09:45:14,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +26: [2023-04-29 09:45:14,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 1: [2023-04-29 09:45:14,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +42: [2023-04-29 09:45:14,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:14,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +36: [2023-04-29 09:45:14,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +36: [2023-04-29 09:45:14,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +12: [2023-04-29 09:45:14,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +36: [2023-04-29 09:45:14,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +36: [2023-04-29 09:45:14,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +17: [2023-04-29 09:45:14,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +12: [2023-04-29 09:45:14,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 1: [2023-04-29 09:45:14,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +50: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +50: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +59: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +59: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +59: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +50: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +20: [2023-04-29 09:45:14,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +59: [2023-04-29 09:45:14,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +12: [2023-04-29 09:45:14,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +36: [2023-04-29 09:45:14,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +12: [2023-04-29 09:45:14,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +51: [2023-04-29 09:45:14,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:14,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +36: [2023-04-29 09:45:14,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:14,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +29: [2023-04-29 09:45:14,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +29: [2023-04-29 09:45:14,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +30: [2023-04-29 09:45:14,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +59: [2023-04-29 09:45:14,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +50: [2023-04-29 09:45:14,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +50: [2023-04-29 09:45:14,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +31: [2023-04-29 09:45:14,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +31: [2023-04-29 09:45:14,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. + 1: [2023-04-29 09:45:14,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +13: [2023-04-29 09:45:14,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +51: [2023-04-29 09:45:14,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:14,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +36: [2023-04-29 09:45:14,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +29: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +12: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +59: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +44: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +58: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_02_optim_states.pt... +58: [2023-04-29 09:45:14,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_02_optim_states.pt... +13: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_02_optim_states.pt... +58: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_02_optim_states.pt... + 1: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +59: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +31: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +12: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +31: [2023-04-29 09:45:14,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +27: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +51: [2023-04-29 09:45:14,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:14,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:14,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +18: [2023-04-29 09:45:14,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +30: [2023-04-29 09:45:14,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +24: [2023-04-29 09:45:14,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +44: [2023-04-29 09:45:14,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:14,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +29: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +28: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +11: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +11: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +11: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +11: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +40: [2023-04-29 09:45:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:14,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:14,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +12: [2023-04-29 09:45:14,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 1: [2023-04-29 09:45:14,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +10: [2023-04-29 09:45:14,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +44: [2023-04-29 09:45:14,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:14,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +21: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +21: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +30: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +40: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +21: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +27: [2023-04-29 09:45:14,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +44: [2023-04-29 09:45:14,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 1: [2023-04-29 09:45:14,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +44: [2023-04-29 09:45:14,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:14,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +11: [2023-04-29 09:45:14,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +11: [2023-04-29 09:45:14,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +11: [2023-04-29 09:45:14,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +53: [2023-04-29 09:45:14,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +11: [2023-04-29 09:45:14,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +53: [2023-04-29 09:45:14,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +53: [2023-04-29 09:45:14,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 1: [2023-04-29 09:45:14,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +21: [2023-04-29 09:45:14,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +29: [2023-04-29 09:45:14,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +53: [2023-04-29 09:45:14,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +10: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +21: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +12: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +24: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +10: [2023-04-29 09:45:14,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +24: [2023-04-29 09:45:14,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:14,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +35: [2023-04-29 09:45:14,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +35: [2023-04-29 09:45:14,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +35: [2023-04-29 09:45:14,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +20: [2023-04-29 09:45:14,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +35: [2023-04-29 09:45:14,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +12: [2023-04-29 09:45:14,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:14,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:14,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +10: [2023-04-29 09:45:14,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:14,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:14,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +35: [2023-04-29 09:45:14,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +20: [2023-04-29 09:45:14,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +20: [2023-04-29 09:45:14,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +35: [2023-04-29 09:45:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:14,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +25: [2023-04-29 09:45:14,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +25: [2023-04-29 09:45:14,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +48: [2023-04-29 09:45:14,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +18: [2023-04-29 09:45:14,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +16: [2023-04-29 09:45:14,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +16: [2023-04-29 09:45:14,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +16: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +31: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +16: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +31: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +31: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +10: [2023-04-29 09:45:14,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +10: [2023-04-29 09:45:14,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +41: [2023-04-29 09:45:14,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +41: [2023-04-29 09:45:14,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +41: [2023-04-29 09:45:14,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +20: [2023-04-29 09:45:14,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +30: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +10: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +58: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +58: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +58: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +27: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +41: [2023-04-29 09:45:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +53: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +53: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +53: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +34: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +42: [2023-04-29 09:45:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +42: [2023-04-29 09:45:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +11: [2023-04-29 09:45:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +42: [2023-04-29 09:45:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +18: [2023-04-29 09:45:14,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +34: [2023-04-29 09:45:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +25: [2023-04-29 09:45:14,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +30: [2023-04-29 09:45:14,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +22: [2023-04-29 09:45:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +30: [2023-04-29 09:45:14,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... + 0: [2023-04-29 09:45:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +18: [2023-04-29 09:45:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +11: [2023-04-29 09:45:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +53: [2023-04-29 09:45:14,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +30: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +56: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +22: [2023-04-29 09:45:14,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +11: [2023-04-29 09:45:14,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +53: [2023-04-29 09:45:14,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +22: [2023-04-29 09:45:14,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +22: [2023-04-29 09:45:14,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +11: [2023-04-29 09:45:14,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +11: [2023-04-29 09:45:14,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +42: [2023-04-29 09:45:14,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:14,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +56: [2023-04-29 09:45:14,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:14,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +18: [2023-04-29 09:45:14,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +56: [2023-04-29 09:45:14,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +11: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 0: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +53: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +53: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +13: [2023-04-29 09:45:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +13: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +13: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +13: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +53: [2023-04-29 09:45:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +56: [2023-04-29 09:45:14,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +11: [2023-04-29 09:45:14,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +11: [2023-04-29 09:45:14,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +13: [2023-04-29 09:45:14,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +13: [2023-04-29 09:45:14,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:14,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +17: [2023-04-29 09:45:14,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +51: [2023-04-29 09:45:14,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_02_optim_states.pt... +51: [2023-04-29 09:45:14,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_02_optim_states.pt... +51: [2023-04-29 09:45:14,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_02_optim_states.pt... +51: [2023-04-29 09:45:14,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_02_optim_states.pt... +36: [2023-04-29 09:45:14,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:14,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:14,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +22: [2023-04-29 09:45:14,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:14,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +43: [2023-04-29 09:45:14,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +43: [2023-04-29 09:45:14,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +34: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +48: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +48: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +17: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +17: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +41: [2023-04-29 09:45:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:14,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +48: [2023-04-29 09:45:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +48: [2023-04-29 09:45:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +48: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +41: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +14: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +14: [2023-04-29 09:45:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +14: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +14: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +41: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +23: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +23: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +23: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +16: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +36: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +44: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +50: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +50: [2023-04-29 09:45:14,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +16: [2023-04-29 09:45:14,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:14,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +34: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +22: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +34: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +59: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +59: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +27: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +58: [2023-04-29 09:45:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +32: [2023-04-29 09:45:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +20: [2023-04-29 09:45:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +17: [2023-04-29 09:45:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +58: [2023-04-29 09:45:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +20: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +59: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +48: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +48: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +42: [2023-04-29 09:45:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +22: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +53: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +53: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +20: [2023-04-29 09:45:14,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:14,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:14,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +59: [2023-04-29 09:45:14,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +61: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +24: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +42: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:14,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +22: [2023-04-29 09:45:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +59: [2023-04-29 09:45:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +22: [2023-04-29 09:45:14,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +20: [2023-04-29 09:45:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +59: [2023-04-29 09:45:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +50: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +34: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +34: [2023-04-29 09:45:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +27: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +61: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_02_optim_states.pt... +59: [2023-04-29 09:45:14,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +57: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +56: [2023-04-29 09:45:14,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +60: [2023-04-29 09:45:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +13: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +23: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:14,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +37: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +37: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +24: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +22: [2023-04-29 09:45:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +30: [2023-04-29 09:45:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +30: [2023-04-29 09:45:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +20: [2023-04-29 09:45:14,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +38: [2023-04-29 09:45:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +38: [2023-04-29 09:45:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_02_optim_states.pt... +48: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +50: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 2: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +35: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:14,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +60: [2023-04-29 09:45:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +13: [2023-04-29 09:45:14,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +23: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +21: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +21: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +37: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +30: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_19-model_01-model_states.pt. +20: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +38: [2023-04-29 09:45:14,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:14,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +44: [2023-04-29 09:45:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +50: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:14,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +60: [2023-04-29 09:45:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +37: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:14,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +48: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_02_optim_states.pt... +48: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +50: [2023-04-29 09:45:14,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +35: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +11: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +60: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +23: [2023-04-29 09:45:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +37: [2023-04-29 09:45:14,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +34: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +36: [2023-04-29 09:45:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +60: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +37: [2023-04-29 09:45:14,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +34: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_02_optim_states.pt... +34: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_02_optim_states.pt... +34: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_02_optim_states.pt... +34: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_02_optim_states.pt... +38: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:14,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +38: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +38: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:14,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:14,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +24: [2023-04-29 09:45:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +42: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:14,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +24: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +42: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +24: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +24: [2023-04-29 09:45:14,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +24: [2023-04-29 09:45:14,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +24: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +25: [2023-04-29 09:45:14,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +28: [2023-04-29 09:45:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +28: [2023-04-29 09:45:14,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:14,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +28: [2023-04-29 09:45:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +28: [2023-04-29 09:45:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +44: [2023-04-29 09:45:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:14,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +31: [2023-04-29 09:45:14,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +44: [2023-04-29 09:45:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:14,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +50: [2023-04-29 09:45:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +53: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_02_optim_states.pt... +53: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_02_optim_states.pt... +53: [2023-04-29 09:45:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_02_optim_states.pt... +36: [2023-04-29 09:45:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +14: [2023-04-29 09:45:14,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +14: [2023-04-29 09:45:14,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +56: [2023-04-29 09:45:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +11: [2023-04-29 09:45:14,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +10: [2023-04-29 09:45:14,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +59: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +59: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +11: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +36: [2023-04-29 09:45:14,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:14,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:14,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:14,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +31: [2023-04-29 09:45:14,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +31: [2023-04-29 09:45:14,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +58: [2023-04-29 09:45:14,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +30: [2023-04-29 09:45:14,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +11: [2023-04-29 09:45:14,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +11: [2023-04-29 09:45:14,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:14,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +13: [2023-04-29 09:45:14,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +30: [2023-04-29 09:45:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +58: [2023-04-29 09:45:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +10: [2023-04-29 09:45:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +62: [2023-04-29 09:45:14,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +46: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +46: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +30: [2023-04-29 09:45:14,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt... +21: [2023-04-29 09:45:14,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +41: [2023-04-29 09:45:14,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +10: [2023-04-29 09:45:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +62: [2023-04-29 09:45:14,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:14,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:14,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +46: [2023-04-29 09:45:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +46: [2023-04-29 09:45:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +46: [2023-04-29 09:45:14,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +47: [2023-04-29 09:45:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +35: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +13: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +41: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +47: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +58: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_02_optim_states.pt... +42: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_02_optim_states.pt... +42: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_02_optim_states.pt... +46: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_02_optim_states.pt... + 7: [2023-04-29 09:45:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +62: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +35: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +16: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +46: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +16: [2023-04-29 09:45:14,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +42: [2023-04-29 09:45:14,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:14,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +16: [2023-04-29 09:45:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +61: [2023-04-29 09:45:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +33: [2023-04-29 09:45:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +47: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +33: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +22: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +22: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +58: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +39: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +39: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +39: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +16: [2023-04-29 09:45:14,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +41: [2023-04-29 09:45:14,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +16: [2023-04-29 09:45:14,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +16: [2023-04-29 09:45:14,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +33: [2023-04-29 09:45:14,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:14,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:14,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:14,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:14,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:14,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +53: [2023-04-29 09:45:14,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +53: [2023-04-29 09:45:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +58: [2023-04-29 09:45:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +10: [2023-04-29 09:45:14,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +39: [2023-04-29 09:45:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +33: [2023-04-29 09:45:14,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:14,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +22: [2023-04-29 09:45:14,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +41: [2023-04-29 09:45:14,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +41: [2023-04-29 09:45:14,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +22: [2023-04-29 09:45:14,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +23: [2023-04-29 09:45:14,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +23: [2023-04-29 09:45:14,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +23: [2023-04-29 09:45:14,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +58: [2023-04-29 09:45:14,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +23: [2023-04-29 09:45:14,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +56: [2023-04-29 09:45:14,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:14,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +53: [2023-04-29 09:45:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +61: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +42: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +41: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +38: [2023-04-29 09:45:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +52: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +52: [2023-04-29 09:45:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +22: [2023-04-29 09:45:14,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +38: [2023-04-29 09:45:14,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +38: [2023-04-29 09:45:14,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +38: [2023-04-29 09:45:14,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +38: [2023-04-29 09:45:14,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +38: [2023-04-29 09:45:14,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:14,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +52: [2023-04-29 09:45:14,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:14,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +42: [2023-04-29 09:45:14,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +52: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +52: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +42: [2023-04-29 09:45:14,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +53: [2023-04-29 09:45:14,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +43: [2023-04-29 09:45:14,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +23: [2023-04-29 09:45:14,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +38: [2023-04-29 09:45:14,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +14: [2023-04-29 09:45:14,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +38: [2023-04-29 09:45:14,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +23: [2023-04-29 09:45:14,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +60: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +10: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +55: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +23: [2023-04-29 09:45:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +43: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 6: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +42: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +13: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:14,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +14: [2023-04-29 09:45:14,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +57: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +39: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +60: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +23: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +25: [2023-04-29 09:45:14,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +39: [2023-04-29 09:45:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +37: [2023-04-29 09:45:14,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +39: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +32: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:14,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +25: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +32: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +23: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +59: [2023-04-29 09:45:14,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +25: [2023-04-29 09:45:14,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +43: [2023-04-29 09:45:14,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:14,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +23: [2023-04-29 09:45:14,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +43: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +48: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +48: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +50: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +51: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +48: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +51: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +51: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +51: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +50: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +50: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +31: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +31: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +61: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +45: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +45: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +37: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +37: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +37: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +23: [2023-04-29 09:45:14,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +45: [2023-04-29 09:45:14,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +62: [2023-04-29 09:45:14,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +31: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +38: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:14,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:14,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:14,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:14,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +32: [2023-04-29 09:45:14,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +32: [2023-04-29 09:45:14,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +32: [2023-04-29 09:45:14,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +32: [2023-04-29 09:45:14,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +54: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +54: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +54: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +33: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +33: [2023-04-29 09:45:14,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +33: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +38: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +47: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +47: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +47: [2023-04-29 09:45:14,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +47: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 5: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +59: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +63: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +63: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +31: [2023-04-29 09:45:14,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +38: [2023-04-29 09:45:14,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +23: [2023-04-29 09:45:14,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +38: [2023-04-29 09:45:14,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +43: [2023-04-29 09:45:14,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +14: [2023-04-29 09:45:14,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +63: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +14: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +63: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +35: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +35: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +35: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +14: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +35: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +30: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +38: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +59: [2023-04-29 09:45:14,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +38: [2023-04-29 09:45:14,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +31: [2023-04-29 09:45:14,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +60: [2023-04-29 09:45:14,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:14,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +34: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +34: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +34: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +63: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:14,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:14,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:14,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:14,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +59: [2023-04-29 09:45:14,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:14,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +57: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +19: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +38: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +19: [2023-04-29 09:45:14,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:14,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +19: [2023-04-29 09:45:14,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +38: [2023-04-29 09:45:14,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:14,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:14,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +25: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +46: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:14,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +59: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +21: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +21: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +21: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +37: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +21: [2023-04-29 09:45:14,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +59: [2023-04-29 09:45:14,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:14,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +30: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +25: [2023-04-29 09:45:14,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +60: [2023-04-29 09:45:14,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:14,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +59: [2023-04-29 09:45:14,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +37: [2023-04-29 09:45:14,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:14,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:14,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:14,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +30: [2023-04-29 09:45:14,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_00-model_states.pt. +41: [2023-04-29 09:45:14,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:14,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +25: [2023-04-29 09:45:14,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:14,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +46: [2023-04-29 09:45:14,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +46: [2023-04-29 09:45:14,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:14,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:14,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +39: [2023-04-29 09:45:14,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +39: [2023-04-29 09:45:14,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +46: [2023-04-29 09:45:14,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 6: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +32: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +41: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +32: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +45: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +60: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +60: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +60: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +49: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +49: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +49: [2023-04-29 09:45:14,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:14,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +49: [2023-04-29 09:45:14,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:14,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:14,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:14,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +37: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:14,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +19: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +30: [2023-04-29 09:45:14,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt... +51: [2023-04-29 09:45:14,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +48: [2023-04-29 09:45:14,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:14,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +49: [2023-04-29 09:45:14,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:14,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:14,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:14,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +45: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +33: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +51: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +48: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +19: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +48: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +51: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +49: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +49: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +47: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:14,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +23: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +37: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +49: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 5: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +35: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +15: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +37: [2023-04-29 09:45:14,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:14,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +48: [2023-04-29 09:45:14,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:14,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:14,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:14,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +51: [2023-04-29 09:45:14,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +54: [2023-04-29 09:45:14,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +57: [2023-04-29 09:45:14,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +15: [2023-04-29 09:45:14,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +57: [2023-04-29 09:45:14,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +57: [2023-04-29 09:45:14,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +35: [2023-04-29 09:45:14,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +57: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +35: [2023-04-29 09:45:14,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +15: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +23: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +34: [2023-04-29 09:45:14,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +34: [2023-04-29 09:45:14,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:14,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +47: [2023-04-29 09:45:14,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +34: [2023-04-29 09:45:14,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 6: [2023-04-29 09:45:14,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 2: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +26: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +26: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +21: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:14,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:14,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:14,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +47: [2023-04-29 09:45:14,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:14,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +25: [2023-04-29 09:45:14,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:14,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +26: [2023-04-29 09:45:14,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +21: [2023-04-29 09:45:14,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +56: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +56: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +21: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +36: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +36: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +46: [2023-04-29 09:45:14,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +36: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +36: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +56: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +18: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:14,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +26: [2023-04-29 09:45:14,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +21: [2023-04-29 09:45:14,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:14,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 6: [2023-04-29 09:45:14,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +43: [2023-04-29 09:45:14,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +52: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +33: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +63: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +63: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +63: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +63: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +26: [2023-04-29 09:45:14,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +46: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +26: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +52: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 8: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 8: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:14,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 9: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:14,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +39: [2023-04-29 09:45:14,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +52: [2023-04-29 09:45:14,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +18: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +62: [2023-04-29 09:45:14,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +41: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +55: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +55: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +40: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +40: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +37: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +37: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +55: [2023-04-29 09:45:14,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +41: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +44: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +60: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +39: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +19: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +44: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +62: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +32: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... +32: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... +32: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... +32: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:14,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +26: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +37: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +46: [2023-04-29 09:45:14,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +38: [2023-04-29 09:45:14,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_02_optim_states.pt... +38: [2023-04-29 09:45:14,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_02_optim_states.pt... +38: [2023-04-29 09:45:14,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_02_optim_states.pt... +38: [2023-04-29 09:45:14,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_02_optim_states.pt... +26: [2023-04-29 09:45:14,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +60: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +33: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +40: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +39: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +33: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +33: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +62: [2023-04-29 09:45:14,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +19: [2023-04-29 09:45:14,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:14,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +46: [2023-04-29 09:45:14,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:14,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +46: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +39: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +39: [2023-04-29 09:45:14,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +55: [2023-04-29 09:45:14,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:14,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:14,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:14,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:14,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:14,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:14,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +31: [2023-04-29 09:45:14,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +39: [2023-04-29 09:45:14,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +54: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 7: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +47: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +15: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +43: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +15: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +23: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:14,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +23: [2023-04-29 09:45:14,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:14,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +41: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:14,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:14,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:14,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +18: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +18: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +54: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +54: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +29: [2023-04-29 09:45:14,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +57: [2023-04-29 09:45:14,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +23: [2023-04-29 09:45:14,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:14,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +29: [2023-04-29 09:45:14,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +47: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +49: [2023-04-29 09:45:14,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +49: [2023-04-29 09:45:14,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +49: [2023-04-29 09:45:14,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +49: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. + 3: [2023-04-29 09:45:14,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +57: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +29: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +38: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +38: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +29: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +19: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +19: [2023-04-29 09:45:14,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 7: [2023-04-29 09:45:14,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 7: [2023-04-29 09:45:14,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +56: [2023-04-29 09:45:14,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +56: [2023-04-29 09:45:14,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:14,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:14,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +26: [2023-04-29 09:45:14,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +56: [2023-04-29 09:45:14,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +56: [2023-04-29 09:45:14,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +31: [2023-04-29 09:45:14,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +43: [2023-04-29 09:45:14,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +38: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +62: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +31: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +36: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +26: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +40: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:14,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +29: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +29: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +36: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 0: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +37: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_02_optim_states.pt... +37: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_02_optim_states.pt... +37: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_02_optim_states.pt... +37: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_02_optim_states.pt... +63: [2023-04-29 09:45:14,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:14,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +63: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +31: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +44: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:14,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +26: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +44: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +33: [2023-04-29 09:45:14,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 8: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +37: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +36: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +55: [2023-04-29 09:45:14,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +29: [2023-04-29 09:45:14,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +63: [2023-04-29 09:45:14,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:14,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +40: [2023-04-29 09:45:14,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +43: [2023-04-29 09:45:14,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:14,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +55: [2023-04-29 09:45:14,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +52: [2023-04-29 09:45:14,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +38: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +29: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +29: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +63: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +29: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +39: [2023-04-29 09:45:14,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:14,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:14,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +63: [2023-04-29 09:45:14,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +43: [2023-04-29 09:45:14,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +45: [2023-04-29 09:45:14,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:14,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +50: [2023-04-29 09:45:14,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:14,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:14,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +43: [2023-04-29 09:45:14,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:14,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +63: [2023-04-29 09:45:14,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:14,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +62: [2023-04-29 09:45:14,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +31: [2023-04-29 09:45:14,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +35: [2023-04-29 09:45:14,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:14,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:14,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +32: [2023-04-29 09:45:14,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +29: [2023-04-29 09:45:14,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:14,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:14,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:14,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +55: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +43: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +55: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +55: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +27: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +32: [2023-04-29 09:45:14,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +32: [2023-04-29 09:45:14,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +45: [2023-04-29 09:45:14,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 3: [2023-04-29 09:45:14,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +45: [2023-04-29 09:45:14,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +51: [2023-04-29 09:45:14,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:14,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 3: [2023-04-29 09:45:14,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +29: [2023-04-29 09:45:14,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:14,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +29: [2023-04-29 09:45:14,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +49: [2023-04-29 09:45:14,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +49: [2023-04-29 09:45:14,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +45: [2023-04-29 09:45:14,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:14,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:14,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:14,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:14,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:14,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:14,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +32: [2023-04-29 09:45:14,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +32: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +18: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +35: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +35: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +30: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +45: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:14,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +34: [2023-04-29 09:45:14,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:14,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:14,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:14,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +33: [2023-04-29 09:45:14,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:14,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +48: [2023-04-29 09:45:14,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +35: [2023-04-29 09:45:14,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +21: [2023-04-29 09:45:14,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +43: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +43: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +49: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +21: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +31: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:14,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +31: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +31: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +41: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_02_optim_states.pt... +41: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_02_optim_states.pt... +30: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +41: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_02_optim_states.pt... +19: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +41: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_02_optim_states.pt... +30: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +34: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +34: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +35: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +15: [2023-04-29 09:45:14,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +35: [2023-04-29 09:45:14,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +30: [2023-04-29 09:45:14,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:14,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +48: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +34: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 2: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +45: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +48: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +21: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +50: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +48: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +51: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +33: [2023-04-29 09:45:14,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 2: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +51: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +47: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +47: [2023-04-29 09:45:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +51: [2023-04-29 09:45:14,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +34: [2023-04-29 09:45:14,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +33: [2023-04-29 09:45:14,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +35: [2023-04-29 09:45:14,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +60: [2023-04-29 09:45:14,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:14,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +54: [2023-04-29 09:45:14,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +21: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 5: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +33: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:14,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +34: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +15: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +34: [2023-04-29 09:45:14,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +46: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +18: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +34: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +19: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +19: [2023-04-29 09:45:14,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +33: [2023-04-29 09:45:14,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +21: [2023-04-29 09:45:14,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +47: [2023-04-29 09:45:14,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +49: [2023-04-29 09:45:14,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 5: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +51: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +60: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 5: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +47: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +63: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +54: [2023-04-29 09:45:14,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +33: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +47: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +21: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +25: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +25: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +19: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:14,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +50: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +47: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +14: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +39: [2023-04-29 09:45:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +14: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +49: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +27: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +39: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:15,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +39: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +46: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +39: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +63: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +33: [2023-04-29 09:45:15,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +25: [2023-04-29 09:45:15,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +50: [2023-04-29 09:45:15,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +46: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +27: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +49: [2023-04-29 09:45:15,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +19: [2023-04-29 09:45:15,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +63: [2023-04-29 09:45:15,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:15,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +33: [2023-04-29 09:45:15,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +21: [2023-04-29 09:45:15,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +57: [2023-04-29 09:45:15,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +60: [2023-04-29 09:45:15,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +14: [2023-04-29 09:45:15,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +54: [2023-04-29 09:45:15,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:15,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:15,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +60: [2023-04-29 09:45:15,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +46: [2023-04-29 09:45:15,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +18: [2023-04-29 09:45:15,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +26: [2023-04-29 09:45:15,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +43: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_02_optim_states.pt... +49: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_02_optim_states.pt... +43: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_02_optim_states.pt... +43: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_02_optim_states.pt... +46: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +26: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +14: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:15,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +27: [2023-04-29 09:45:15,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +46: [2023-04-29 09:45:15,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +18: [2023-04-29 09:45:15,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:15,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:15,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 2: [2023-04-29 09:45:15,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:15,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +27: [2023-04-29 09:45:15,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +26: [2023-04-29 09:45:15,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:15,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:15,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:15,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:15,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:15,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:15,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:15,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +56: [2023-04-29 09:45:15,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +46: [2023-04-29 09:45:15,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:15,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:15,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +54: [2023-04-29 09:45:15,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +18: [2023-04-29 09:45:15,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +57: [2023-04-29 09:45:15,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +15: [2023-04-29 09:45:15,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +15: [2023-04-29 09:45:15,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +15: [2023-04-29 09:45:15,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +27: [2023-04-29 09:45:15,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +27: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +49: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +26: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +56: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:15,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:15,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +27: [2023-04-29 09:45:15,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:15,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +37: [2023-04-29 09:45:15,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:15,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:15,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:15,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +18: [2023-04-29 09:45:15,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +62: [2023-04-29 09:45:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:15,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +44: [2023-04-29 09:45:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:15,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +54: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +56: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +40: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:15,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +57: [2023-04-29 09:45:15,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 9: [2023-04-29 09:45:15,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +36: [2023-04-29 09:45:15,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +25: [2023-04-29 09:45:15,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +25: [2023-04-29 09:45:15,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +15: [2023-04-29 09:45:15,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +62: [2023-04-29 09:45:15,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +44: [2023-04-29 09:45:15,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +44: [2023-04-29 09:45:15,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +56: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +52: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +44: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +52: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +16: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +56: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +63: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +25: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +40: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +16: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +54: [2023-04-29 09:45:15,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +54: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +16: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +36: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +49: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +37: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +49: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +44: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +29: [2023-04-29 09:45:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +49: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +49: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +19: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +26: [2023-04-29 09:45:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:15,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +31: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +36: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +44: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +26: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +17: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +44: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +37: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +37: [2023-04-29 09:45:15,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +55: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +55: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +63: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +24: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +24: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +24: [2023-04-29 09:45:15,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +24: [2023-04-29 09:45:15,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +17: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +39: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_02_optim_states.pt... +39: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_02_optim_states.pt... +39: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_02_optim_states.pt... +31: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +39: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_02_optim_states.pt... + 4: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +31: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +52: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +52: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +37: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +44: [2023-04-29 09:45:15,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +25: [2023-04-29 09:45:15,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +63: [2023-04-29 09:45:15,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +63: [2023-04-29 09:45:15,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +40: [2023-04-29 09:45:15,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +40: [2023-04-29 09:45:15,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +40: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +40: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +36: [2023-04-29 09:45:15,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +18: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +52: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +19: [2023-04-29 09:45:15,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +37: [2023-04-29 09:45:15,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +37: [2023-04-29 09:45:15,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +24: [2023-04-29 09:45:15,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +63: [2023-04-29 09:45:15,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +26: [2023-04-29 09:45:15,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +36: [2023-04-29 09:45:15,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +28: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +62: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +28: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +24: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +26: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +55: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +55: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +24: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +55: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +55: [2023-04-29 09:45:15,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +24: [2023-04-29 09:45:15,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:15,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +63: [2023-04-29 09:45:15,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +12: [2023-04-29 09:45:15,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +63: [2023-04-29 09:45:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +26: [2023-04-29 09:45:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:15,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +29: [2023-04-29 09:45:15,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:15,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:15,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +29: [2023-04-29 09:45:15,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +29: [2023-04-29 09:45:15,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +29: [2023-04-29 09:45:15,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:15,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:15,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:15,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:15,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +63: [2023-04-29 09:45:15,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +28: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_02_optim_states.pt... +49: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_02_optim_states.pt... +49: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_02_optim_states.pt... +49: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_02_optim_states.pt... +52: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +62: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +12: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +12: [2023-04-29 09:45:15,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +49: [2023-04-29 09:45:15,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +26: [2023-04-29 09:45:15,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:15,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +29: [2023-04-29 09:45:15,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +19: [2023-04-29 09:45:15,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +29: [2023-04-29 09:45:15,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +29: [2023-04-29 09:45:15,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +49: [2023-04-29 09:45:15,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +49: [2023-04-29 09:45:15,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +15: [2023-04-29 09:45:15,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:15,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:15,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +26: [2023-04-29 09:45:15,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +62: [2023-04-29 09:45:15,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +15: [2023-04-29 09:45:15,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +49: [2023-04-29 09:45:15,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +30: [2023-04-29 09:45:15,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:15,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:15,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +19: [2023-04-29 09:45:15,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +49: [2023-04-29 09:45:15,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +49: [2023-04-29 09:45:15,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +31: [2023-04-29 09:45:15,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +31: [2023-04-29 09:45:15,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +29: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +54: [2023-04-29 09:45:15,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:15,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:15,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +18: [2023-04-29 09:45:15,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +19: [2023-04-29 09:45:15,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:15,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:15,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:15,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +54: [2023-04-29 09:45:15,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:15,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +27: [2023-04-29 09:45:15,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +30: [2023-04-29 09:45:15,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +29: [2023-04-29 09:45:15,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +54: [2023-04-29 09:45:15,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +18: [2023-04-29 09:45:15,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +49: [2023-04-29 09:45:15,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +14: [2023-04-29 09:45:15,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +18: [2023-04-29 09:45:15,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +27: [2023-04-29 09:45:15,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +18: [2023-04-29 09:45:15,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +29: [2023-04-29 09:45:15,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +29: [2023-04-29 09:45:15,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +29: [2023-04-29 09:45:15,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:15,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +30: [2023-04-29 09:45:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +14: [2023-04-29 09:45:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +17: [2023-04-29 09:45:15,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +17: [2023-04-29 09:45:15,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +17: [2023-04-29 09:45:15,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +17: [2023-04-29 09:45:15,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:15,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +25: [2023-04-29 09:45:15,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +15: [2023-04-29 09:45:15,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +28: [2023-04-29 09:45:15,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +17: [2023-04-29 09:45:15,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +17: [2023-04-29 09:45:15,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +17: [2023-04-29 09:45:15,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:15,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +12: [2023-04-29 09:45:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +25: [2023-04-29 09:45:15,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +25: [2023-04-29 09:45:15,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +24: [2023-04-29 09:45:15,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +25: [2023-04-29 09:45:15,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +14: [2023-04-29 09:45:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +24: [2023-04-29 09:45:15,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +16: [2023-04-29 09:45:15,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +24: [2023-04-29 09:45:15,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +24: [2023-04-29 09:45:15,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:15,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +16: [2023-04-29 09:45:15,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +28: [2023-04-29 09:45:15,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +45: [2023-04-29 09:45:15,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_02_optim_states.pt. +45: [2023-04-29 09:45:15,123] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 360 +50: [2023-04-29 09:45:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:15,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +12: [2023-04-29 09:45:15,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +40: [2023-04-29 09:45:15,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +30: [2023-04-29 09:45:15,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +30: [2023-04-29 09:45:15,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +40: [2023-04-29 09:45:15,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +31: [2023-04-29 09:45:15,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:15,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +50: [2023-04-29 09:45:15,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:15,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:15,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:15,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +30: [2023-04-29 09:45:15,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_20-model_01-model_states.pt. +24: [2023-04-29 09:45:15,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:15,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +17: [2023-04-29 09:45:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +45: [2023-04-29 09:45:15,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_02_optim_states.pt. + 4: [2023-04-29 09:45:15,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +45: [2023-04-29 09:45:15,136] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 364 +31: [2023-04-29 09:45:15,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +31: [2023-04-29 09:45:15,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +31: [2023-04-29 09:45:15,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +16: [2023-04-29 09:45:15,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +16: [2023-04-29 09:45:15,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +28: [2023-04-29 09:45:15,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +17: [2023-04-29 09:45:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +31: [2023-04-29 09:45:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +40: [2023-04-29 09:45:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +17: [2023-04-29 09:45:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +17: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_02_optim_states.pt. +58: [2023-04-29 09:45:15,146] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 466 +24: [2023-04-29 09:45:15,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +24: [2023-04-29 09:45:15,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +24: [2023-04-29 09:45:15,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:15,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +17: [2023-04-29 09:45:15,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +17: [2023-04-29 09:45:15,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +30: [2023-04-29 09:45:15,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... +17: [2023-04-29 09:45:15,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +44: [2023-04-29 09:45:15,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:15,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:15,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:15,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:15,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:15,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:15,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:15,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_02_optim_states.pt... +40: [2023-04-29 09:45:15,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_02_optim_states.pt... +40: [2023-04-29 09:45:15,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_02_optim_states.pt... +40: [2023-04-29 09:45:15,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_02_optim_states.pt... +17: [2023-04-29 09:45:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +44: [2023-04-29 09:45:15,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:15,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:15,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:15,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:15,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:15,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:15,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:15,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_02_optim_states.pt. +45: [2023-04-29 09:45:15,173] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 362 +44: [2023-04-29 09:45:15,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +36: [2023-04-29 09:45:15,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +36: [2023-04-29 09:45:15,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:15,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +17: [2023-04-29 09:45:15,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +22: [2023-04-29 09:45:15,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +22: [2023-04-29 09:45:15,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +22: [2023-04-29 09:45:15,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +22: [2023-04-29 09:45:15,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +20: [2023-04-29 09:45:15,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +59: [2023-04-29 09:45:15,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +59: [2023-04-29 09:45:15,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:15,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:15,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +22: [2023-04-29 09:45:15,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +22: [2023-04-29 09:45:15,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +22: [2023-04-29 09:45:15,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +35: [2023-04-29 09:45:15,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:15,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:15,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:15,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +20: [2023-04-29 09:45:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +35: [2023-04-29 09:45:15,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +12: [2023-04-29 09:45:15,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +12: [2023-04-29 09:45:15,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +12: [2023-04-29 09:45:15,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +12: [2023-04-29 09:45:15,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +35: [2023-04-29 09:45:15,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:15,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:15,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:15,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +17: [2023-04-29 09:45:15,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +17: [2023-04-29 09:45:15,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +34: [2023-04-29 09:45:15,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_02_optim_states.pt. +34: [2023-04-29 09:45:15,206] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 278 +17: [2023-04-29 09:45:15,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +51: [2023-04-29 09:45:15,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_02_optim_states.pt. +51: [2023-04-29 09:45:15,206] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 414 +50: [2023-04-29 09:45:15,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +41: [2023-04-29 09:45:15,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +41: [2023-04-29 09:45:15,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +41: [2023-04-29 09:45:15,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +41: [2023-04-29 09:45:15,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +48: [2023-04-29 09:45:15,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_02_optim_states.pt. +48: [2023-04-29 09:45:15,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 386 +34: [2023-04-29 09:45:15,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_02_optim_states.pt. +34: [2023-04-29 09:45:15,215] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 276 +12: [2023-04-29 09:45:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +48: [2023-04-29 09:45:15,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_02_optim_states.pt. +48: [2023-04-29 09:45:15,219] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 388 +12: [2023-04-29 09:45:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +51: [2023-04-29 09:45:15,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_02_optim_states.pt. +12: [2023-04-29 09:45:15,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +51: [2023-04-29 09:45:15,222] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 408 +12: [2023-04-29 09:45:15,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +11: [2023-04-29 09:45:15,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +45: [2023-04-29 09:45:15,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_02_optim_states.pt. +11: [2023-04-29 09:45:15,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +11: [2023-04-29 09:45:15,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +45: [2023-04-29 09:45:15,225] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 366 +11: [2023-04-29 09:45:15,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +50: [2023-04-29 09:45:15,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:15,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +30: [2023-04-29 09:45:15,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:15,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +30: [2023-04-29 09:45:15,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +30: [2023-04-29 09:45:15,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_00-model_states.pt. +50: [2023-04-29 09:45:15,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:15,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_02_optim_states.pt. +58: [2023-04-29 09:45:15,231] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 470 +50: [2023-04-29 09:45:15,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:15,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +30: [2023-04-29 09:45:15,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... + 7: [2023-04-29 09:45:15,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +30: [2023-04-29 09:45:15,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +50: [2023-04-29 09:45:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +30: [2023-04-29 09:45:15,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt... +50: [2023-04-29 09:45:15,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +61: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +61: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +50: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +48: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_02_optim_states.pt. +61: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_41-model_01-model_states.pt. +50: [2023-04-29 09:45:15,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:15,238] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 384 +44: [2023-04-29 09:45:15,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:15,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:15,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:15,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:15,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:15,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +20: [2023-04-29 09:45:15,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:15,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +22: [2023-04-29 09:45:15,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +22: [2023-04-29 09:45:15,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:15,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:15,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +60: [2023-04-29 09:45:15,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:15,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:15,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +22: [2023-04-29 09:45:15,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +35: [2023-04-29 09:45:15,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +21: [2023-04-29 09:45:15,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +59: [2023-04-29 09:45:15,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +11: [2023-04-29 09:45:15,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +21: [2023-04-29 09:45:15,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +21: [2023-04-29 09:45:15,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +36: [2023-04-29 09:45:15,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +21: [2023-04-29 09:45:15,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +21: [2023-04-29 09:45:15,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +35: [2023-04-29 09:45:15,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +44: [2023-04-29 09:45:15,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:15,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_02_optim_states.pt. +53: [2023-04-29 09:45:15,265] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 424 +35: [2023-04-29 09:45:15,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:15,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +61: [2023-04-29 09:45:15,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +34: [2023-04-29 09:45:15,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_02_optim_states.pt. +34: [2023-04-29 09:45:15,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 274 +20: [2023-04-29 09:45:15,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +13: [2023-04-29 09:45:15,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +61: [2023-04-29 09:45:15,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +13: [2023-04-29 09:45:15,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +13: [2023-04-29 09:45:15,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +61: [2023-04-29 09:45:15,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt... +44: [2023-04-29 09:45:15,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:15,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:15,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_02_optim_states.pt... +50: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_02_optim_states.pt... +50: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:15,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:15,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:15,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:15,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +25: [2023-04-29 09:45:15,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +22: [2023-04-29 09:45:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +22: [2023-04-29 09:45:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:15,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:15,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:15,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +25: [2023-04-29 09:45:15,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +25: [2023-04-29 09:45:15,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:15,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_02_optim_states.pt. +53: [2023-04-29 09:45:15,283] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 428 +20: [2023-04-29 09:45:15,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:15,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +20: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +59: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +59: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +25: [2023-04-29 09:45:15,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:15,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +59: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +12: [2023-04-29 09:45:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +36: [2023-04-29 09:45:15,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +12: [2023-04-29 09:45:15,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +12: [2023-04-29 09:45:15,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +35: [2023-04-29 09:45:15,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:15,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +13: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +35: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +10: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +35: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +12: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +11: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +11: [2023-04-29 09:45:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:15,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +42: [2023-04-29 09:45:15,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_02_optim_states.pt. +11: [2023-04-29 09:45:15,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +42: [2023-04-29 09:45:15,295] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 336 +11: [2023-04-29 09:45:15,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +58: [2023-04-29 09:45:15,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +58: [2023-04-29 09:45:15,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +58: [2023-04-29 09:45:15,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +58: [2023-04-29 09:45:15,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +12: [2023-04-29 09:45:15,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +13: [2023-04-29 09:45:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +42: [2023-04-29 09:45:15,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +42: [2023-04-29 09:45:15,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +42: [2023-04-29 09:45:15,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +42: [2023-04-29 09:45:15,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +59: [2023-04-29 09:45:15,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +59: [2023-04-29 09:45:15,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +60: [2023-04-29 09:45:15,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +44: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_02_optim_states.pt... +44: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_02_optim_states.pt... + 6: [2023-04-29 09:45:15,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +60: [2023-04-29 09:45:15,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:15,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:15,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:15,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +21: [2023-04-29 09:45:15,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +41: [2023-04-29 09:45:15,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:15,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +41: [2023-04-29 09:45:15,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +41: [2023-04-29 09:45:15,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +24: [2023-04-29 09:45:15,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +24: [2023-04-29 09:45:15,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +11: [2023-04-29 09:45:15,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +24: [2023-04-29 09:45:15,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +24: [2023-04-29 09:45:15,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +23: [2023-04-29 09:45:15,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:15,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +53: [2023-04-29 09:45:15,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +53: [2023-04-29 09:45:15,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +53: [2023-04-29 09:45:15,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +36: [2023-04-29 09:45:15,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_02_optim_states.pt... +36: [2023-04-29 09:45:15,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_02_optim_states.pt... +36: [2023-04-29 09:45:15,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_02_optim_states.pt... +36: [2023-04-29 09:45:15,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_02_optim_states.pt... +32: [2023-04-29 09:45:15,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +32: [2023-04-29 09:45:15,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +41: [2023-04-29 09:45:15,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +53: [2023-04-29 09:45:15,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_02_optim_states.pt. +32: [2023-04-29 09:45:15,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +53: [2023-04-29 09:45:15,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 426 +32: [2023-04-29 09:45:15,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +41: [2023-04-29 09:45:15,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +41: [2023-04-29 09:45:15,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:15,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_02_optim_states.pt. +58: [2023-04-29 09:45:15,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +20: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +20: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +42: [2023-04-29 09:45:15,328] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 342 +42: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +11: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +11: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +58: [2023-04-29 09:45:15,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:15,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +58: [2023-04-29 09:45:15,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:15,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +21: [2023-04-29 09:45:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +59: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +61: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:15,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +59: [2023-04-29 09:45:15,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +59: [2023-04-29 09:45:15,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +41: [2023-04-29 09:45:15,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +11: [2023-04-29 09:45:15,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +11: [2023-04-29 09:45:15,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +11: [2023-04-29 09:45:15,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +60: [2023-04-29 09:45:15,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:15,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +61: [2023-04-29 09:45:15,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +61: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +61: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +59: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_02_optim_states.pt... +35: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_02_optim_states.pt... +57: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_02_optim_states.pt... +61: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_02_optim_states.pt... +57: [2023-04-29 09:45:15,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:15,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:15,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_02_optim_states.pt... +59: [2023-04-29 09:45:15,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_02_optim_states.pt... +59: [2023-04-29 09:45:15,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_02_optim_states.pt... +59: [2023-04-29 09:45:15,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_02_optim_states.pt... +61: [2023-04-29 09:45:15,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +61: [2023-04-29 09:45:15,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_00-model_states.pt. +24: [2023-04-29 09:45:15,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:15,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +24: [2023-04-29 09:45:15,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +61: [2023-04-29 09:45:15,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +61: [2023-04-29 09:45:15,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt... +62: [2023-04-29 09:45:15,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:15,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:15,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:15,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +14: [2023-04-29 09:45:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +53: [2023-04-29 09:45:15,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +14: [2023-04-29 09:45:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +14: [2023-04-29 09:45:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +14: [2023-04-29 09:45:15,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +62: [2023-04-29 09:45:15,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:15,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +24: [2023-04-29 09:45:15,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +24: [2023-04-29 09:45:15,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +53: [2023-04-29 09:45:15,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:15,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +32: [2023-04-29 09:45:15,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:15,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:15,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +10: [2023-04-29 09:45:15,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +10: [2023-04-29 09:45:15,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +10: [2023-04-29 09:45:15,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +32: [2023-04-29 09:45:15,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:15,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:15,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +53: [2023-04-29 09:45:15,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:15,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +20: [2023-04-29 09:45:15,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +42: [2023-04-29 09:45:15,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_02_optim_states.pt. +42: [2023-04-29 09:45:15,368] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 338 + 7: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +11: [2023-04-29 09:45:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +23: [2023-04-29 09:45:15,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +23: [2023-04-29 09:45:15,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +23: [2023-04-29 09:45:15,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +25: [2023-04-29 09:45:15,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +14: [2023-04-29 09:45:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +14: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:15,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +38: [2023-04-29 09:45:15,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +38: [2023-04-29 09:45:15,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +38: [2023-04-29 09:45:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +28: [2023-04-29 09:45:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +28: [2023-04-29 09:45:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_02_optim_states.pt. +38: [2023-04-29 09:45:15,389] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 310 +58: [2023-04-29 09:45:15,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:15,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +23: [2023-04-29 09:45:15,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +47: [2023-04-29 09:45:15,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_02_optim_states.pt... +47: [2023-04-29 09:45:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +60: [2023-04-29 09:45:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_02_optim_states.pt... +60: [2023-04-29 09:45:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_02_optim_states.pt... +60: [2023-04-29 09:45:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_02_optim_states.pt... +42: [2023-04-29 09:45:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +11: [2023-04-29 09:45:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +10: [2023-04-29 09:45:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +10: [2023-04-29 09:45:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +39: [2023-04-29 09:45:15,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +39: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +39: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +13: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +10: [2023-04-29 09:45:15,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +58: [2023-04-29 09:45:15,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +13: [2023-04-29 09:45:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +39: [2023-04-29 09:45:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +10: [2023-04-29 09:45:15,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +23: [2023-04-29 09:45:15,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +42: [2023-04-29 09:45:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:15,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 6: [2023-04-29 09:45:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +58: [2023-04-29 09:45:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +58: [2023-04-29 09:45:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +23: [2023-04-29 09:45:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +37: [2023-04-29 09:45:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_02_optim_states.pt. +37: [2023-04-29 09:45:15,399] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 298 +59: [2023-04-29 09:45:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +42: [2023-04-29 09:45:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +38: [2023-04-29 09:45:15,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_02_optim_states.pt. +13: [2023-04-29 09:45:15,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +38: [2023-04-29 09:45:15,400] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 304 + 6: [2023-04-29 09:45:15,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +11: [2023-04-29 09:45:15,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +61: [2023-04-29 09:45:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:15,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +58: [2023-04-29 09:45:15,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:15,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +42: [2023-04-29 09:45:15,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +13: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +43: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +43: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +43: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +59: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +23: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +13: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_02_optim_states.pt. +43: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +58: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +43: [2023-04-29 09:45:15,403] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 348 + 6: [2023-04-29 09:45:15,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +24: [2023-04-29 09:45:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:15,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +42: [2023-04-29 09:45:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +23: [2023-04-29 09:45:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +23: [2023-04-29 09:45:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +58: [2023-04-29 09:45:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +61: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +45: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +45: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +57: [2023-04-29 09:45:15,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:15,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:15,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:15,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +29: [2023-04-29 09:45:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +29: [2023-04-29 09:45:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +42: [2023-04-29 09:45:15,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +24: [2023-04-29 09:45:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +24: [2023-04-29 09:45:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +56: [2023-04-29 09:45:15,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:15,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:15,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:15,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +32: [2023-04-29 09:45:15,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. +46: [2023-04-29 09:45:15,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:15,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +29: [2023-04-29 09:45:15,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +32: [2023-04-29 09:45:15,413] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 262 +57: [2023-04-29 09:45:15,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +24: [2023-04-29 09:45:15,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +29: [2023-04-29 09:45:15,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:15,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +38: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +29: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +46: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. + 6: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +29: [2023-04-29 09:45:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +10: [2023-04-29 09:45:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +32: [2023-04-29 09:45:15,417] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 258 +38: [2023-04-29 09:45:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:15,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:15,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +38: [2023-04-29 09:45:15,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:15,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:15,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:15,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:15,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +26: [2023-04-29 09:45:15,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +42: [2023-04-29 09:45:15,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_02_optim_states.pt. +39: [2023-04-29 09:45:15,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +42: [2023-04-29 09:45:15,420] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 340 +32: [2023-04-29 09:45:15,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +23: [2023-04-29 09:45:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +53: [2023-04-29 09:45:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +53: [2023-04-29 09:45:15,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +20: [2023-04-29 09:45:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +46: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:15,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:15,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +59: [2023-04-29 09:45:15,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +24: [2023-04-29 09:45:15,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +47: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +47: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +39: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +39: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +26: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +53: [2023-04-29 09:45:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +24: [2023-04-29 09:45:15,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:15,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:15,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +32: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +23: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +50: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +50: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +52: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +52: [2023-04-29 09:45:15,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:15,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +43: [2023-04-29 09:45:15,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +61: [2023-04-29 09:45:15,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +16: [2023-04-29 09:45:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +32: [2023-04-29 09:45:15,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +24: [2023-04-29 09:45:15,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 6: [2023-04-29 09:45:15,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +45: [2023-04-29 09:45:15,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +53: [2023-04-29 09:45:15,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +37: [2023-04-29 09:45:15,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_02_optim_states.pt. +32: [2023-04-29 09:45:15,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +37: [2023-04-29 09:45:15,433] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 300 +24: [2023-04-29 09:45:15,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 6: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +61: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +18: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +53: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +32: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +18: [2023-04-29 09:45:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +32: [2023-04-29 09:45:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +43: [2023-04-29 09:45:15,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_02_optim_states.pt. +43: [2023-04-29 09:45:15,435] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 350 +43: [2023-04-29 09:45:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +20: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +16: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +16: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +16: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +43: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +43: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +45: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +14: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +61: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +61: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +18: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +61: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +54: [2023-04-29 09:45:15,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +61: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_02_optim_states.pt. +18: [2023-04-29 09:45:15,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +61: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:15,440] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 302 +57: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +61: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +57: [2023-04-29 09:45:15,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +18: [2023-04-29 09:45:15,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +57: [2023-04-29 09:45:15,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +18: [2023-04-29 09:45:15,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:15,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +28: [2023-04-29 09:45:15,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +57: [2023-04-29 09:45:15,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +19: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +19: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +19: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +19: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +57: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +57: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +14: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +41: [2023-04-29 09:45:15,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_02_optim_states.pt. +41: [2023-04-29 09:45:15,444] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 332 +28: [2023-04-29 09:45:15,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +56: [2023-04-29 09:45:15,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +56: [2023-04-29 09:45:15,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +28: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +28: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +20: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +56: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +54: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +54: [2023-04-29 09:45:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +56: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +51: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +51: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +51: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +51: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +55: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +10: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +62: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +19: [2023-04-29 09:45:15,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:15,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:15,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +33: [2023-04-29 09:45:15,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +33: [2023-04-29 09:45:15,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +33: [2023-04-29 09:45:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +19: [2023-04-29 09:45:15,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:15,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:15,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:15,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +62: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +30: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +62: [2023-04-29 09:45:15,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +20: [2023-04-29 09:45:15,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +10: [2023-04-29 09:45:15,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +50: [2023-04-29 09:45:15,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +30: [2023-04-29 09:45:15,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +54: [2023-04-29 09:45:15,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:15,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:15,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +54: [2023-04-29 09:45:15,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:15,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:15,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +20: [2023-04-29 09:45:15,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +47: [2023-04-29 09:45:15,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:15,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +54: [2023-04-29 09:45:15,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +54: [2023-04-29 09:45:15,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:15,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:15,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +48: [2023-04-29 09:45:15,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +48: [2023-04-29 09:45:15,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +48: [2023-04-29 09:45:15,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +48: [2023-04-29 09:45:15,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +49: [2023-04-29 09:45:15,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_02_optim_states.pt. +49: [2023-04-29 09:45:15,462] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 396 +30: [2023-04-29 09:45:15,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +10: [2023-04-29 09:45:15,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +16: [2023-04-29 09:45:15,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +50: [2023-04-29 09:45:15,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:15,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +13: [2023-04-29 09:45:15,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +50: [2023-04-29 09:45:15,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +41: [2023-04-29 09:45:15,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_02_optim_states.pt. +23: [2023-04-29 09:45:15,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +41: [2023-04-29 09:45:15,467] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 328 +54: [2023-04-29 09:45:15,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +50: [2023-04-29 09:45:15,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +22: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +54: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +22: [2023-04-29 09:45:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +23: [2023-04-29 09:45:15,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +17: [2023-04-29 09:45:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +47: [2023-04-29 09:45:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:15,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +17: [2023-04-29 09:45:15,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:15,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +10: [2023-04-29 09:45:15,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +14: [2023-04-29 09:45:15,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +39: [2023-04-29 09:45:15,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +38: [2023-04-29 09:45:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +54: [2023-04-29 09:45:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +17: [2023-04-29 09:45:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +23: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +29: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +51: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +13: [2023-04-29 09:45:15,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +15: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +15: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +17: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +15: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +41: [2023-04-29 09:45:15,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_02_optim_states.pt. + 8: [2023-04-29 09:45:15,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +39: [2023-04-29 09:45:15,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +41: [2023-04-29 09:45:15,480] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 334 +28: [2023-04-29 09:45:15,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_02_optim_states.pt... +62: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_02_optim_states.pt... +62: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_02_optim_states.pt... +62: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_02_optim_states.pt... +35: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +35: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +51: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +23: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +51: [2023-04-29 09:45:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 2: [2023-04-29 09:45:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +54: [2023-04-29 09:45:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +28: [2023-04-29 09:45:15,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +38: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +61: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_02_optim_states.pt... +61: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_02_optim_states.pt... +47: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_02_optim_states.pt... +61: [2023-04-29 09:45:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_02_optim_states.pt... + 6: [2023-04-29 09:45:15,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +46: [2023-04-29 09:45:15,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:15,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +38: [2023-04-29 09:45:15,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +51: [2023-04-29 09:45:15,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:15,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +35: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +55: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +55: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +55: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +35: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +55: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +48: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +38: [2023-04-29 09:45:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +23: [2023-04-29 09:45:15,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +23: [2023-04-29 09:45:15,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +13: [2023-04-29 09:45:15,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +28: [2023-04-29 09:45:15,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +47: [2023-04-29 09:45:15,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:15,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +24: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +24: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +24: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +47: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +28: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +52: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +24: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +46: [2023-04-29 09:45:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +38: [2023-04-29 09:45:15,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +38: [2023-04-29 09:45:15,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 8: [2023-04-29 09:45:15,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +26: [2023-04-29 09:45:15,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +48: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +28: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +22: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +52: [2023-04-29 09:45:15,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:15,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:15,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_02_optim_states.pt... +56: [2023-04-29 09:45:15,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_02_optim_states.pt... +56: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_02_optim_states.pt... +56: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_02_optim_states.pt... +22: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +38: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +26: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +29: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +43: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +28: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +48: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +39: [2023-04-29 09:45:15,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +24: [2023-04-29 09:45:15,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +48: [2023-04-29 09:45:15,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:15,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +18: [2023-04-29 09:45:15,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +39: [2023-04-29 09:45:15,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +39: [2023-04-29 09:45:15,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 9: [2023-04-29 09:45:15,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 9: [2023-04-29 09:45:15,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +45: [2023-04-29 09:45:15,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +24: [2023-04-29 09:45:15,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:15,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_02_optim_states.pt... +57: [2023-04-29 09:45:15,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_02_optim_states.pt... +57: [2023-04-29 09:45:15,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_02_optim_states.pt... +57: [2023-04-29 09:45:15,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_02_optim_states.pt... +22: [2023-04-29 09:45:15,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +24: [2023-04-29 09:45:15,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +24: [2023-04-29 09:45:15,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +43: [2023-04-29 09:45:15,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +45: [2023-04-29 09:45:15,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +39: [2023-04-29 09:45:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 7: [2023-04-29 09:45:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 7: [2023-04-29 09:45:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 7: [2023-04-29 09:45:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +39: [2023-04-29 09:45:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +39: [2023-04-29 09:45:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 6: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +18: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +31: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +43: [2023-04-29 09:45:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +31: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +47: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +19: [2023-04-29 09:45:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +45: [2023-04-29 09:45:15,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +19: [2023-04-29 09:45:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +15: [2023-04-29 09:45:15,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +18: [2023-04-29 09:45:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +45: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +19: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 6: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +18: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +49: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_02_optim_states.pt. +43: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +45: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +49: [2023-04-29 09:45:15,509] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 398 +31: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +31: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +31: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +26: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +27: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +31: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +13: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +35: [2023-04-29 09:45:15,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +27: [2023-04-29 09:45:15,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +27: [2023-04-29 09:45:15,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +43: [2023-04-29 09:45:15,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +35: [2023-04-29 09:45:15,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 8: [2023-04-29 09:45:15,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +45: [2023-04-29 09:45:15,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +28: [2023-04-29 09:45:15,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +33: [2023-04-29 09:45:15,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +33: [2023-04-29 09:45:15,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +33: [2023-04-29 09:45:15,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +15: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +57: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +33: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +45: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +57: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +57: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +57: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +47: [2023-04-29 09:45:15,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +47: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +34: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +33: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +47: [2023-04-29 09:45:15,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +43: [2023-04-29 09:45:15,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +43: [2023-04-29 09:45:15,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +16: [2023-04-29 09:45:15,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +33: [2023-04-29 09:45:15,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +55: [2023-04-29 09:45:15,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +35: [2023-04-29 09:45:15,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:15,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +21: [2023-04-29 09:45:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +21: [2023-04-29 09:45:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 7: [2023-04-29 09:45:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +43: [2023-04-29 09:45:15,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +35: [2023-04-29 09:45:15,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:15,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:15,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_02_optim_states.pt. +41: [2023-04-29 09:45:15,526] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 330 +19: [2023-04-29 09:45:15,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:15,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +29: [2023-04-29 09:45:15,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +29: [2023-04-29 09:45:15,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +29: [2023-04-29 09:45:15,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +30: [2023-04-29 09:45:15,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +50: [2023-04-29 09:45:15,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +29: [2023-04-29 09:45:15,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 9: [2023-04-29 09:45:15,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +50: [2023-04-29 09:45:15,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +60: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +60: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +60: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +52: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +47: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_02_optim_states.pt... +47: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_02_optim_states.pt... +47: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_02_optim_states.pt... +47: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_02_optim_states.pt... +38: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_02_optim_states.pt. +37: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +37: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +38: [2023-04-29 09:45:15,533] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 308 +49: [2023-04-29 09:45:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_02_optim_states.pt. +49: [2023-04-29 09:45:15,533] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 392 +18: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +52: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +37: [2023-04-29 09:45:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +37: [2023-04-29 09:45:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +26: [2023-04-29 09:45:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +55: [2023-04-29 09:45:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +39: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_02_optim_states.pt. +39: [2023-04-29 09:45:15,537] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 312 +16: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +25: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +27: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +26: [2023-04-29 09:45:15,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +12: [2023-04-29 09:45:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +54: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +26: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +47: [2023-04-29 09:45:15,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +47: [2023-04-29 09:45:15,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +27: [2023-04-29 09:45:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +28: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +34: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +17: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_02_optim_states.pt. +38: [2023-04-29 09:45:15,544] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 306 +34: [2023-04-29 09:45:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +34: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +27: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +54: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +54: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +54: [2023-04-29 09:45:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_02_optim_states.pt. +51: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +12: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +39: [2023-04-29 09:45:15,545] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 314 +12: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +46: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +46: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +34: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +50: [2023-04-29 09:45:15,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:15,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +12: [2023-04-29 09:45:15,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +47: [2023-04-29 09:45:15,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +19: [2023-04-29 09:45:15,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +62: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +16: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +33: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:15,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +63: [2023-04-29 09:45:15,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +63: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +54: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +54: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +17: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +50: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +54: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 2: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +54: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +14: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +47: [2023-04-29 09:45:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +47: [2023-04-29 09:45:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +16: [2023-04-29 09:45:15,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +21: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +46: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +46: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +39: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_02_optim_states.pt. +39: [2023-04-29 09:45:15,550] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 318 +19: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +19: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +19: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +62: [2023-04-29 09:45:15,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +14: [2023-04-29 09:45:15,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:15,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +33: [2023-04-29 09:45:15,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +51: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +33: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +33: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +20: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +20: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +14: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +47: [2023-04-29 09:45:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +21: [2023-04-29 09:45:15,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +47: [2023-04-29 09:45:15,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +50: [2023-04-29 09:45:15,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +21: [2023-04-29 09:45:15,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 3: [2023-04-29 09:45:15,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 3: [2023-04-29 09:45:15,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 3: [2023-04-29 09:45:15,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +29: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +50: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +51: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:15,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +16: [2023-04-29 09:45:15,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +20: [2023-04-29 09:45:15,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +48: [2023-04-29 09:45:15,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +29: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +33: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +29: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +15: [2023-04-29 09:45:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +33: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +33: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +24: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +37: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +37: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +24: [2023-04-29 09:45:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +51: [2023-04-29 09:45:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +60: [2023-04-29 09:45:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +20: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +29: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +20: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +15: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +37: [2023-04-29 09:45:15,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +33: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +33: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +22: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +51: [2023-04-29 09:45:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +20: [2023-04-29 09:45:15,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +15: [2023-04-29 09:45:15,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +48: [2023-04-29 09:45:15,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +51: [2023-04-29 09:45:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +15: [2023-04-29 09:45:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +60: [2023-04-29 09:45:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +60: [2023-04-29 09:45:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +51: [2023-04-29 09:45:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +30: [2023-04-29 09:45:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +15: [2023-04-29 09:45:15,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +55: [2023-04-29 09:45:15,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +48: [2023-04-29 09:45:15,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 388 +22: [2023-04-29 09:45:15,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +48: [2023-04-29 09:45:15,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:15,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:15,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:15,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +22: [2023-04-29 09:45:15,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +22: [2023-04-29 09:45:15,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +25: [2023-04-29 09:45:15,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +24: [2023-04-29 09:45:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +24: [2023-04-29 09:45:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +55: [2023-04-29 09:45:15,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:15,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +22: [2023-04-29 09:45:15,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +51: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +31: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +55: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:15,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +25: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +48: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +46: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:15,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +63: [2023-04-29 09:45:15,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:15,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +44: [2023-04-29 09:45:15,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +44: [2023-04-29 09:45:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +63: [2023-04-29 09:45:15,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +22: [2023-04-29 09:45:15,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +46: [2023-04-29 09:45:15,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +63: [2023-04-29 09:45:15,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +46: [2023-04-29 09:45:15,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +56: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +31: [2023-04-29 09:45:15,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +62: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 9: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +56: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +62: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +15: [2023-04-29 09:45:15,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +35: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +35: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +48: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +46: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_02_optim_states.pt... +46: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_02_optim_states.pt... +46: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_02_optim_states.pt... +46: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_02_optim_states.pt... + 7: [2023-04-29 09:45:15,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +48: [2023-04-29 09:45:15,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +31: [2023-04-29 09:45:15,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +31: [2023-04-29 09:45:15,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +40: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +29: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +15: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +16: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +54: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +29: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... +29: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +29: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +48: [2023-04-29 09:45:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +52: [2023-04-29 09:45:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_02_optim_states.pt... +52: [2023-04-29 09:45:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_02_optim_states.pt... +40: [2023-04-29 09:45:15,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +52: [2023-04-29 09:45:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_02_optim_states.pt... +52: [2023-04-29 09:45:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_02_optim_states.pt... +40: [2023-04-29 09:45:15,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +54: [2023-04-29 09:45:15,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:15,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +48: [2023-04-29 09:45:15,586] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 386 + 2: [2023-04-29 09:45:15,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +35: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +35: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 9: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +31: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +15: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +54: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:15,588] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 384 +54: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +19: [2023-04-29 09:45:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,589] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 398 + 5: [2023-04-29 09:45:15,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +19: [2023-04-29 09:45:15,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +19: [2023-04-29 09:45:15,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +28: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +16: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +49: [2023-04-29 09:45:15,591] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 396 +16: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +16: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... + 3: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +16: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +16: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +35: [2023-04-29 09:45:15,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +19: [2023-04-29 09:45:15,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... +15: [2023-04-29 09:45:15,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 9: [2023-04-29 09:45:15,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +49: [2023-04-29 09:45:15,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 392 +12: [2023-04-29 09:45:15,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:15,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +28: [2023-04-29 09:45:15,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +30: [2023-04-29 09:45:15,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +30: [2023-04-29 09:45:15,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +55: [2023-04-29 09:45:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +55: [2023-04-29 09:45:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +19: [2023-04-29 09:45:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +15: [2023-04-29 09:45:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +33: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_02_optim_states.pt... +33: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_02_optim_states.pt... +33: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_02_optim_states.pt... +33: [2023-04-29 09:45:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_02_optim_states.pt... + 1: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +30: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 1: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +30: [2023-04-29 09:45:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +35: [2023-04-29 09:45:15,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +58: [2023-04-29 09:45:15,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_02_optim_states.pt. +19: [2023-04-29 09:45:15,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +58: [2023-04-29 09:45:15,600] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 468 +19: [2023-04-29 09:45:15,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 2: [2023-04-29 09:45:15,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:15,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +35: [2023-04-29 09:45:15,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +22: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +22: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +22: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +22: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +19: [2023-04-29 09:45:15,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +26: [2023-04-29 09:45:15,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +44: [2023-04-29 09:45:15,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +55: [2023-04-29 09:45:15,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +55: [2023-04-29 09:45:15,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +55: [2023-04-29 09:45:15,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +55: [2023-04-29 09:45:15,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +21: [2023-04-29 09:45:15,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 9: [2023-04-29 09:45:15,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +34: [2023-04-29 09:45:15,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +44: [2023-04-29 09:45:15,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:15,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. + 7: [2023-04-29 09:45:15,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +44: [2023-04-29 09:45:15,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +32: [2023-04-29 09:45:15,608] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 260 +40: [2023-04-29 09:45:15,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +44: [2023-04-29 09:45:15,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:15,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +26: [2023-04-29 09:45:15,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +22: [2023-04-29 09:45:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +22: [2023-04-29 09:45:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +35: [2023-04-29 09:45:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +22: [2023-04-29 09:45:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:15,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +37: [2023-04-29 09:45:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +26: [2023-04-29 09:45:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +17: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 9: [2023-04-29 09:45:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +40: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +34: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +22: [2023-04-29 09:45:15,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +56: [2023-04-29 09:45:15,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +12: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +37: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +17: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +12: [2023-04-29 09:45:15,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +56: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +57: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +34: [2023-04-29 09:45:15,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +26: [2023-04-29 09:45:15,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +21: [2023-04-29 09:45:15,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +57: [2023-04-29 09:45:15,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +29: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +56: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +34: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +40: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_02_optim_states.pt. + 9: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +34: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +40: [2023-04-29 09:45:15,618] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 326 +29: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +57: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +57: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +37: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +27: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +12: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:15,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +31: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +31: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +31: [2023-04-29 09:45:15,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +34: [2023-04-29 09:45:15,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +31: [2023-04-29 09:45:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +31: [2023-04-29 09:45:15,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +29: [2023-04-29 09:45:15,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +17: [2023-04-29 09:45:15,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +29: [2023-04-29 09:45:15,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +27: [2023-04-29 09:45:15,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +21: [2023-04-29 09:45:15,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +27: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +37: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +14: [2023-04-29 09:45:15,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +17: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +57: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +14: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +39: [2023-04-29 09:45:15,624] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 312 +30: [2023-04-29 09:45:15,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +57: [2023-04-29 09:45:15,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +15: [2023-04-29 09:45:15,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +57: [2023-04-29 09:45:15,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +27: [2023-04-29 09:45:15,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +21: [2023-04-29 09:45:15,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +33: [2023-04-29 09:45:15,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +30: [2023-04-29 09:45:15,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +29: [2023-04-29 09:45:15,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +20: [2023-04-29 09:45:15,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +51: [2023-04-29 09:45:15,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_02_optim_states.pt. +55: [2023-04-29 09:45:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_02_optim_states.pt... +55: [2023-04-29 09:45:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_02_optim_states.pt... +55: [2023-04-29 09:45:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_02_optim_states.pt... +55: [2023-04-29 09:45:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_02_optim_states.pt... +51: [2023-04-29 09:45:15,628] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 412 +30: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +27: [2023-04-29 09:45:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +27: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +37: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +33: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +60: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +60: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +27: [2023-04-29 09:45:15,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +25: [2023-04-29 09:45:15,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +33: [2023-04-29 09:45:15,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +37: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +24: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +24: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +25: [2023-04-29 09:45:15,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:15,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +25: [2023-04-29 09:45:15,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +46: [2023-04-29 09:45:15,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:15,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +21: [2023-04-29 09:45:15,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +15: [2023-04-29 09:45:15,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +37: [2023-04-29 09:45:15,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 4: [2023-04-29 09:45:15,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +60: [2023-04-29 09:45:15,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +60: [2023-04-29 09:45:15,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +15: [2023-04-29 09:45:15,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +37: [2023-04-29 09:45:15,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +54: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_02_optim_states.pt... +54: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_02_optim_states.pt... +54: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_02_optim_states.pt... +54: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_02_optim_states.pt... +29: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +33: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +25: [2023-04-29 09:45:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +19: [2023-04-29 09:45:15,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:15,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +20: [2023-04-29 09:45:15,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:15,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +60: [2023-04-29 09:45:15,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +46: [2023-04-29 09:45:15,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +63: [2023-04-29 09:45:15,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +25: [2023-04-29 09:45:15,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +46: [2023-04-29 09:45:15,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +17: [2023-04-29 09:45:15,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +17: [2023-04-29 09:45:15,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +60: [2023-04-29 09:45:15,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +33: [2023-04-29 09:45:15,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +33: [2023-04-29 09:45:15,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +17: [2023-04-29 09:45:15,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +46: [2023-04-29 09:45:15,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +20: [2023-04-29 09:45:15,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +52: [2023-04-29 09:45:15,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +52: [2023-04-29 09:45:15,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 3: [2023-04-29 09:45:15,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 4: [2023-04-29 09:45:15,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +62: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +15: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +25: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +46: [2023-04-29 09:45:15,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +40: [2023-04-29 09:45:15,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_02_optim_states.pt. +20: [2023-04-29 09:45:15,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +40: [2023-04-29 09:45:15,647] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 322 +63: [2023-04-29 09:45:15,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +63: [2023-04-29 09:45:15,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +19: [2023-04-29 09:45:15,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +52: [2023-04-29 09:45:15,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +19: [2023-04-29 09:45:15,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +39: [2023-04-29 09:45:15,649] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 314 +49: [2023-04-29 09:45:15,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +49: [2023-04-29 09:45:15,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +46: [2023-04-29 09:45:15,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +52: [2023-04-29 09:45:15,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +46: [2023-04-29 09:45:15,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +60: [2023-04-29 09:45:15,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 0: [2023-04-29 09:45:15,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +15: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +60: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +49: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +46: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +63: [2023-04-29 09:45:15,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +61: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +61: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +18: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +62: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +61: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +61: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 3: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 4: [2023-04-29 09:45:15,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +36: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +36: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. +63: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +63: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +63: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +62: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +62: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +25: [2023-04-29 09:45:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_21-model_01-model_states.pt. + 4: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... + 4: [2023-04-29 09:45:15,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... + 7: [2023-04-29 09:45:15,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... + 7: [2023-04-29 09:45:15,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... + 7: [2023-04-29 09:45:15,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... + 7: [2023-04-29 09:45:15,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 3: [2023-04-29 09:45:15,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +19: [2023-04-29 09:45:15,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +63: [2023-04-29 09:45:15,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +36: [2023-04-29 09:45:15,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. +36: [2023-04-29 09:45:15,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_42-model_01-model_states.pt. + 5: [2023-04-29 09:45:15,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 5: [2023-04-29 09:45:15,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +39: [2023-04-29 09:45:15,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 318 +15: [2023-04-29 09:45:15,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:15,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +62: [2023-04-29 09:45:15,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +62: [2023-04-29 09:45:15,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +25: [2023-04-29 09:45:15,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +22: [2023-04-29 09:45:15,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +25: [2023-04-29 09:45:15,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +15: [2023-04-29 09:45:15,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +12: [2023-04-29 09:45:15,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +12: [2023-04-29 09:45:15,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +12: [2023-04-29 09:45:15,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... + 5: [2023-04-29 09:45:15,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +40: [2023-04-29 09:45:15,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:15,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +10: [2023-04-29 09:45:15,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +10: [2023-04-29 09:45:15,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +10: [2023-04-29 09:45:15,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +62: [2023-04-29 09:45:15,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +40: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +10: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... + 1: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... + 1: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... + 5: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +49: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +22: [2023-04-29 09:45:15,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +52: [2023-04-29 09:45:15,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +49: [2023-04-29 09:45:15,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:15,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +22: [2023-04-29 09:45:15,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +56: [2023-04-29 09:45:15,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:15,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +10: [2023-04-29 09:45:15,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +10: [2023-04-29 09:45:15,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +61: [2023-04-29 09:45:15,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +40: [2023-04-29 09:45:15,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +40: [2023-04-29 09:45:15,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +52: [2023-04-29 09:45:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +14: [2023-04-29 09:45:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +14: [2023-04-29 09:45:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-04-29 09:45:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +14: [2023-04-29 09:45:15,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... + 0: [2023-04-29 09:45:15,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +44: [2023-04-29 09:45:15,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:15,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +61: [2023-04-29 09:45:15,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:15,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +61: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +52: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +40: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:15,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +44: [2023-04-29 09:45:15,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:15,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +22: [2023-04-29 09:45:15,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +40: [2023-04-29 09:45:15,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +44: [2023-04-29 09:45:15,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +44: [2023-04-29 09:45:15,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +31: [2023-04-29 09:45:15,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +63: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +36: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +40: [2023-04-29 09:45:15,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +44: [2023-04-29 09:45:15,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +56: [2023-04-29 09:45:15,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +30: [2023-04-29 09:45:15,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +18: [2023-04-29 09:45:15,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +44: [2023-04-29 09:45:15,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +63: [2023-04-29 09:45:15,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:15,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:15,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +56: [2023-04-29 09:45:15,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +31: [2023-04-29 09:45:15,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +44: [2023-04-29 09:45:15,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +18: [2023-04-29 09:45:15,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt... +44: [2023-04-29 09:45:15,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 6: [2023-04-29 09:45:15,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:15,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +56: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +27: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +27: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +56: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +36: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt... +63: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:15,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +31: [2023-04-29 09:45:15,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +30: [2023-04-29 09:45:15,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +21: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +12: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +12: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +63: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +12: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +63: [2023-04-29 09:45:15,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +11: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +11: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +11: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +11: [2023-04-29 09:45:15,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +25: [2023-04-29 09:45:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +56: [2023-04-29 09:45:15,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:15,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +30: [2023-04-29 09:45:15,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +30: [2023-04-29 09:45:15,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +58: [2023-04-29 09:45:15,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_02_optim_states.pt. +58: [2023-04-29 09:45:15,704] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 464 +25: [2023-04-29 09:45:15,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +20: [2023-04-29 09:45:15,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +20: [2023-04-29 09:45:15,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +20: [2023-04-29 09:45:15,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +20: [2023-04-29 09:45:15,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +30: [2023-04-29 09:45:15,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +30: [2023-04-29 09:45:15,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +17: [2023-04-29 09:45:15,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:15,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 1: [2023-04-29 09:45:15,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +17: [2023-04-29 09:45:15,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +17: [2023-04-29 09:45:15,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +25: [2023-04-29 09:45:15,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:15,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +37: [2023-04-29 09:45:15,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_02_optim_states.pt. +37: [2023-04-29 09:45:15,721] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 296 +17: [2023-04-29 09:45:15,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 1: [2023-04-29 09:45:15,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +11: [2023-04-29 09:45:15,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:15,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +12: [2023-04-29 09:45:15,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +12: [2023-04-29 09:45:15,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:15,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +12: [2023-04-29 09:45:15,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +53: [2023-04-29 09:45:15,727] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 428 + 3: [2023-04-29 09:45:15,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +11: [2023-04-29 09:45:15,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +10: [2023-04-29 09:45:15,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:15,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +23: [2023-04-29 09:45:15,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +49: [2023-04-29 09:45:15,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +23: [2023-04-29 09:45:15,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +18: [2023-04-29 09:45:15,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +49: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +13: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +11: [2023-04-29 09:45:15,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +23: [2023-04-29 09:45:15,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +49: [2023-04-29 09:45:15,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +52: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +53: [2023-04-29 09:45:15,743] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 426 +22: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +18: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +22: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +22: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... +61: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +13: [2023-04-29 09:45:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +53: [2023-04-29 09:45:15,747] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 424 +50: [2023-04-29 09:45:15,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_02_optim_states.pt. +50: [2023-04-29 09:45:15,748] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 406 +50: [2023-04-29 09:45:15,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_02_optim_states.pt. +50: [2023-04-29 09:45:15,748] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 402 +49: [2023-04-29 09:45:15,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:15,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +49: [2023-04-29 09:45:15,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +36: [2023-04-29 09:45:15,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +52: [2023-04-29 09:45:15,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +10: [2023-04-29 09:45:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +10: [2023-04-29 09:45:15,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +10: [2023-04-29 09:45:15,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +49: [2023-04-29 09:45:15,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +61: [2023-04-29 09:45:15,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:15,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +53: [2023-04-29 09:45:15,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_02_optim_states.pt. +49: [2023-04-29 09:45:15,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +53: [2023-04-29 09:45:15,756] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 430 +61: [2023-04-29 09:45:15,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +36: [2023-04-29 09:45:15,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +44: [2023-04-29 09:45:15,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_02_optim_states.pt. +52: [2023-04-29 09:45:15,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +44: [2023-04-29 09:45:15,759] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 352 +61: [2023-04-29 09:45:15,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 0: [2023-04-29 09:45:15,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +26: [2023-04-29 09:45:15,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:15,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +61: [2023-04-29 09:45:15,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +53: [2023-04-29 09:45:15,762] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 430 +61: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +63: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +36: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +36: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +18: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +18: [2023-04-29 09:45:15,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:15,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... + 0: [2023-04-29 09:45:15,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +52: [2023-04-29 09:45:15,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +61: [2023-04-29 09:45:15,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +44: [2023-04-29 09:45:15,766] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 352 + 6: [2023-04-29 09:45:15,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:15,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_00-model_states.pt. +26: [2023-04-29 09:45:15,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +36: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +36: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_00-model_states.pt. +21: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... +26: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +21: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +36: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +21: [2023-04-29 09:45:15,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... + 3: [2023-04-29 09:45:15,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +52: [2023-04-29 09:45:15,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +52: [2023-04-29 09:45:15,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +42: [2023-04-29 09:45:15,771] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 340 +63: [2023-04-29 09:45:15,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:15,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +18: [2023-04-29 09:45:15,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +18: [2023-04-29 09:45:15,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +44: [2023-04-29 09:45:15,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_02_optim_states.pt. +44: [2023-04-29 09:45:15,775] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 358 +25: [2023-04-29 09:45:15,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-04-29 09:45:15,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +25: [2023-04-29 09:45:15,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +25: [2023-04-29 09:45:15,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +36: [2023-04-29 09:45:15,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt... +18: [2023-04-29 09:45:15,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt... +44: [2023-04-29 09:45:15,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_02_optim_states.pt. +44: [2023-04-29 09:45:15,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 356 +44: [2023-04-29 09:45:15,782] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 358 +51: [2023-04-29 09:45:15,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_02_optim_states.pt. +51: [2023-04-29 09:45:15,784] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 410 +11: [2023-04-29 09:45:15,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +35: [2023-04-29 09:45:15,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_02_optim_states.pt. +44: [2023-04-29 09:45:15,785] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 356 + 0: [2023-04-29 09:45:15,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +35: [2023-04-29 09:45:15,785] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 280 + 0: [2023-04-29 09:45:15,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +12: [2023-04-29 09:45:15,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 0: [2023-04-29 09:45:15,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +63: [2023-04-29 09:45:15,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:15,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:15,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_02_optim_states.pt. +36: [2023-04-29 09:45:15,792] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 294 +12: [2023-04-29 09:45:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +34: [2023-04-29 09:45:15,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_02_optim_states.pt. +34: [2023-04-29 09:45:15,796] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 272 +11: [2023-04-29 09:45:15,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +63: [2023-04-29 09:45:15,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:15,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +23: [2023-04-29 09:45:15,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +63: [2023-04-29 09:45:15,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:15,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +12: [2023-04-29 09:45:15,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +63: [2023-04-29 09:45:15,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:15,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:15,801] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 294 +63: [2023-04-29 09:45:15,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:15,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:15,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:15,802] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 342 +11: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +63: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +11: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:15,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:15,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +20: [2023-04-29 09:45:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +20: [2023-04-29 09:45:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +20: [2023-04-29 09:45:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +20: [2023-04-29 09:45:15,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +42: [2023-04-29 09:45:15,807] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 336 +42: [2023-04-29 09:45:15,807] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 338 +12: [2023-04-29 09:45:15,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 8: [2023-04-29 09:45:15,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:15,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +11: [2023-04-29 09:45:15,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:15,812] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 310 + 9: [2023-04-29 09:45:15,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:15,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +23: [2023-04-29 09:45:15,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +50: [2023-04-29 09:45:15,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_02_optim_states.pt. +38: [2023-04-29 09:45:15,815] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 306 +50: [2023-04-29 09:45:15,815] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 400 +48: [2023-04-29 09:45:15,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_02_optim_states.pt. +48: [2023-04-29 09:45:15,816] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 390 +10: [2023-04-29 09:45:15,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +10: [2023-04-29 09:45:15,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +10: [2023-04-29 09:45:15,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-04-29 09:45:15,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +59: [2023-04-29 09:45:15,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_02_optim_states.pt. +59: [2023-04-29 09:45:15,817] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 478 +11: [2023-04-29 09:45:15,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:15,818] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 304 +11: [2023-04-29 09:45:15,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +36: [2023-04-29 09:45:15,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_02_optim_states.pt. +11: [2023-04-29 09:45:15,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +36: [2023-04-29 09:45:15,819] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 292 +38: [2023-04-29 09:45:15,821] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 308 +59: [2023-04-29 09:45:15,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_02_optim_states.pt. +59: [2023-04-29 09:45:15,821] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 474 +59: [2023-04-29 09:45:15,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_02_optim_states.pt. +59: [2023-04-29 09:45:15,823] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 472 +48: [2023-04-29 09:45:15,824] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 390 +18: [2023-04-29 09:45:15,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +36: [2023-04-29 09:45:15,827] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 292 +43: [2023-04-29 09:45:15,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_02_optim_states.pt. +43: [2023-04-29 09:45:15,828] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 344 +24: [2023-04-29 09:45:15,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +24: [2023-04-29 09:45:15,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +24: [2023-04-29 09:45:15,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +24: [2023-04-29 09:45:15,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +18: [2023-04-29 09:45:15,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +32: [2023-04-29 09:45:15,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. +32: [2023-04-29 09:45:15,832] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 256 +20: [2023-04-29 09:45:15,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +18: [2023-04-29 09:45:15,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +18: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +44: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_02_optim_states.pt. + 6: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... + 6: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... + 6: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +44: [2023-04-29 09:45:15,835] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 354 +26: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:15,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +60: [2023-04-29 09:45:15,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_02_optim_states.pt. +60: [2023-04-29 09:45:15,837] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 480 +26: [2023-04-29 09:45:15,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +53: [2023-04-29 09:45:15,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +53: [2023-04-29 09:45:15,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +53: [2023-04-29 09:45:15,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +53: [2023-04-29 09:45:15,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +20: [2023-04-29 09:45:15,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +44: [2023-04-29 09:45:15,842] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 354 +19: [2023-04-29 09:45:15,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +19: [2023-04-29 09:45:15,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +19: [2023-04-29 09:45:15,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +20: [2023-04-29 09:45:15,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +20: [2023-04-29 09:45:15,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +60: [2023-04-29 09:45:15,844] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 480 + 5: [2023-04-29 09:45:15,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +63: [2023-04-29 09:45:15,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_02_optim_states.pt... +63: [2023-04-29 09:45:15,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_02_optim_states.pt... +63: [2023-04-29 09:45:15,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_02_optim_states.pt... +63: [2023-04-29 09:45:15,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_02_optim_states.pt... +10: [2023-04-29 09:45:15,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +10: [2023-04-29 09:45:15,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +10: [2023-04-29 09:45:15,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +10: [2023-04-29 09:45:15,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 0: [2023-04-29 09:45:15,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +60: [2023-04-29 09:45:15,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_02_optim_states.pt. +60: [2023-04-29 09:45:15,851] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 486 +43: [2023-04-29 09:45:15,853] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 350 + 2: [2023-04-29 09:45:15,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +45: [2023-04-29 09:45:15,854] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 362 +45: [2023-04-29 09:45:15,854] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 366 +35: [2023-04-29 09:45:15,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_02_optim_states.pt. +35: [2023-04-29 09:45:15,855] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 284 +30: [2023-04-29 09:45:15,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:15,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +60: [2023-04-29 09:45:15,858] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 486 +40: [2023-04-29 09:45:15,857] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 322 +24: [2023-04-29 09:45:15,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +40: [2023-04-29 09:45:15,859] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 326 +43: [2023-04-29 09:45:15,859] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 344 + 0: [2023-04-29 09:45:15,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +45: [2023-04-29 09:45:15,860] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 360 +24: [2023-04-29 09:45:15,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +45: [2023-04-29 09:45:15,864] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 364 +24: [2023-04-29 09:45:15,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +24: [2023-04-29 09:45:15,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 0: [2023-04-29 09:45:15,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:15,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +43: [2023-04-29 09:45:15,868] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 348 +53: [2023-04-29 09:45:15,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:15,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:15,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +53: [2023-04-29 09:45:15,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:15,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_02_optim_states.pt. +59: [2023-04-29 09:45:15,873] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 476 +13: [2023-04-29 09:45:15,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +13: [2023-04-29 09:45:15,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-04-29 09:45:15,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +53: [2023-04-29 09:45:15,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:15,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +10: [2023-04-29 09:45:15,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +14: [2023-04-29 09:45:15,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +14: [2023-04-29 09:45:15,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +11: [2023-04-29 09:45:15,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +11: [2023-04-29 09:45:15,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +11: [2023-04-29 09:45:15,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +11: [2023-04-29 09:45:15,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +10: [2023-04-29 09:45:15,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +58: [2023-04-29 09:45:15,880] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 468 +23: [2023-04-29 09:45:15,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +23: [2023-04-29 09:45:15,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +23: [2023-04-29 09:45:15,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +23: [2023-04-29 09:45:15,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +10: [2023-04-29 09:45:15,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:15,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +28: [2023-04-29 09:45:15,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +28: [2023-04-29 09:45:15,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +28: [2023-04-29 09:45:15,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +28: [2023-04-29 09:45:15,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +31: [2023-04-29 09:45:15,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:15,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +58: [2023-04-29 09:45:15,885] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 470 +35: [2023-04-29 09:45:15,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_02_optim_states.pt. +58: [2023-04-29 09:45:15,885] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 466 +35: [2023-04-29 09:45:15,885] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 282 +58: [2023-04-29 09:45:15,886] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 464 +31: [2023-04-29 09:45:15,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:15,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 8: [2023-04-29 09:45:15,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... + 8: [2023-04-29 09:45:15,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-04-29 09:45:15,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +15: [2023-04-29 09:45:15,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:15,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +43: [2023-04-29 09:45:15,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_02_optim_states.pt. +18: [2023-04-29 09:45:15,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +32: [2023-04-29 09:45:15,893] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 262 +43: [2023-04-29 09:45:15,893] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 346 +15: [2023-04-29 09:45:15,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +19: [2023-04-29 09:45:15,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +18: [2023-04-29 09:45:15,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +43: [2023-04-29 09:45:15,901] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 346 +20: [2023-04-29 09:45:15,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +18: [2023-04-29 09:45:15,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:15,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +39: [2023-04-29 09:45:15,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_02_optim_states.pt. +39: [2023-04-29 09:45:15,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 316 +32: [2023-04-29 09:45:15,906] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 260 +20: [2023-04-29 09:45:15,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +41: [2023-04-29 09:45:15,909] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 328 +39: [2023-04-29 09:45:15,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 316 +26: [2023-04-29 09:45:15,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +26: [2023-04-29 09:45:15,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... +26: [2023-04-29 09:45:15,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +26: [2023-04-29 09:45:15,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +13: [2023-04-29 09:45:15,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +13: [2023-04-29 09:45:15,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +13: [2023-04-29 09:45:15,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +14: [2023-04-29 09:45:15,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +14: [2023-04-29 09:45:15,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:15,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 8: [2023-04-29 09:45:15,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +32: [2023-04-29 09:45:15,922] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 258 +30: [2023-04-29 09:45:15,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:15,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +32: [2023-04-29 09:45:15,923] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 256 +28: [2023-04-29 09:45:15,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:15,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +24: [2023-04-29 09:45:15,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +28: [2023-04-29 09:45:15,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +41: [2023-04-29 09:45:15,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:15,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:15,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:15,927] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 330 +24: [2023-04-29 09:45:15,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +41: [2023-04-29 09:45:15,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:15,929] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 334 + 2: [2023-04-29 09:45:15,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +53: [2023-04-29 09:45:15,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +20: [2023-04-29 09:45:15,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +24: [2023-04-29 09:45:15,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +41: [2023-04-29 09:45:15,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 332 +30: [2023-04-29 09:45:15,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +24: [2023-04-29 09:45:15,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 2: [2023-04-29 09:45:15,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +53: [2023-04-29 09:45:15,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:15,933] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 406 +30: [2023-04-29 09:45:15,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +23: [2023-04-29 09:45:15,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +23: [2023-04-29 09:45:15,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +23: [2023-04-29 09:45:15,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +24: [2023-04-29 09:45:15,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:15,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +53: [2023-04-29 09:45:15,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +58: [2023-04-29 09:45:15,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +58: [2023-04-29 09:45:15,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +24: [2023-04-29 09:45:15,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +58: [2023-04-29 09:45:15,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +58: [2023-04-29 09:45:15,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +61: [2023-04-29 09:45:15,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_02_optim_states.pt. +10: [2023-04-29 09:45:15,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +61: [2023-04-29 09:45:15,938] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 492 +53: [2023-04-29 09:45:15,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 0: [2023-04-29 09:45:15,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-04-29 09:45:15,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 0: [2023-04-29 09:45:15,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... + 0: [2023-04-29 09:45:15,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +53: [2023-04-29 09:45:15,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:15,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:15,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:15,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +56: [2023-04-29 09:45:15,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_02_optim_states.pt. +56: [2023-04-29 09:45:15,941] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 448 +61: [2023-04-29 09:45:15,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_02_optim_states.pt. +61: [2023-04-29 09:45:15,941] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 490 +24: [2023-04-29 09:45:15,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +10: [2023-04-29 09:45:15,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +24: [2023-04-29 09:45:15,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +10: [2023-04-29 09:45:15,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +61: [2023-04-29 09:45:15,945] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 492 +53: [2023-04-29 09:45:15,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +13: [2023-04-29 09:45:15,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:15,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +53: [2023-04-29 09:45:15,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +10: [2023-04-29 09:45:15,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... + 9: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 9: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +27: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:15,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +51: [2023-04-29 09:45:15,947] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 412 +27: [2023-04-29 09:45:15,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:15,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +61: [2023-04-29 09:45:15,948] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 490 +13: [2023-04-29 09:45:15,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,949] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 402 +56: [2023-04-29 09:45:15,949] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 448 +10: [2023-04-29 09:45:15,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 6: [2023-04-29 09:45:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 6: [2023-04-29 09:45:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 6: [2023-04-29 09:45:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 6: [2023-04-29 09:45:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +10: [2023-04-29 09:45:15,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 8: [2023-04-29 09:45:15,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:15,952] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 400 +53: [2023-04-29 09:45:15,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 8: [2023-04-29 09:45:15,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 8: [2023-04-29 09:45:15,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:15,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +37: [2023-04-29 09:45:15,955] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 298 +37: [2023-04-29 09:45:15,956] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 302 +31: [2023-04-29 09:45:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +41: [2023-04-29 09:45:15,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:15,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +36: [2023-04-29 09:45:15,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_02_optim_states.pt. + 8: [2023-04-29 09:45:15,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +36: [2023-04-29 09:45:15,959] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 288 +10: [2023-04-29 09:45:15,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +41: [2023-04-29 09:45:15,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 7: [2023-04-29 09:45:15,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 2: [2023-04-29 09:45:15,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +31: [2023-04-29 09:45:15,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +58: [2023-04-29 09:45:15,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:15,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:15,965] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 296 + 7: [2023-04-29 09:45:15,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +15: [2023-04-29 09:45:15,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +10: [2023-04-29 09:45:15,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +36: [2023-04-29 09:45:15,967] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 288 +23: [2023-04-29 09:45:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +58: [2023-04-29 09:45:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +23: [2023-04-29 09:45:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +58: [2023-04-29 09:45:15,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:15,968] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 300 +58: [2023-04-29 09:45:15,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +15: [2023-04-29 09:45:15,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:15,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:15,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:15,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +34: [2023-04-29 09:45:15,972] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 272 + 3: [2023-04-29 09:45:15,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +51: [2023-04-29 09:45:15,973] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 414 + 3: [2023-04-29 09:45:15,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:15,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +34: [2023-04-29 09:45:15,974] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 274 +51: [2023-04-29 09:45:15,975] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 410 + 6: [2023-04-29 09:45:15,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +51: [2023-04-29 09:45:15,976] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 408 +21: [2023-04-29 09:45:15,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +21: [2023-04-29 09:45:15,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +21: [2023-04-29 09:45:15,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +21: [2023-04-29 09:45:15,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +47: [2023-04-29 09:45:15,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_02_optim_states.pt. +47: [2023-04-29 09:45:15,978] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 378 +47: [2023-04-29 09:45:15,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_02_optim_states.pt. +47: [2023-04-29 09:45:15,978] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 382 + 3: [2023-04-29 09:45:15,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:15,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_02_optim_states.pt. +57: [2023-04-29 09:45:15,980] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 460 + 6: [2023-04-29 09:45:15,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:15,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 3: [2023-04-29 09:45:15,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:15,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 3: [2023-04-29 09:45:15,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:15,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... + 5: [2023-04-29 09:45:15,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-04-29 09:45:15,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... + 5: [2023-04-29 09:45:15,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +61: [2023-04-29 09:45:15,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_02_optim_states.pt. +61: [2023-04-29 09:45:15,987] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 488 +57: [2023-04-29 09:45:15,987] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 460 +18: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +18: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +18: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +19: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +19: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... +56: [2023-04-29 09:45:15,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_02_optim_states.pt. +56: [2023-04-29 09:45:15,989] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 454 +34: [2023-04-29 09:45:15,990] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 276 +14: [2023-04-29 09:45:15,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +61: [2023-04-29 09:45:15,994] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 488 +28: [2023-04-29 09:45:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +16: [2023-04-29 09:45:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +16: [2023-04-29 09:45:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +56: [2023-04-29 09:45:15,995] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 454 +16: [2023-04-29 09:45:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +14: [2023-04-29 09:45:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +34: [2023-04-29 09:45:15,997] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 278 +13: [2023-04-29 09:45:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +42: [2023-04-29 09:45:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +42: [2023-04-29 09:45:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +42: [2023-04-29 09:45:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +62: [2023-04-29 09:45:15,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_02_optim_states.pt. +42: [2023-04-29 09:45:15,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +62: [2023-04-29 09:45:15,999] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 502 +14: [2023-04-29 09:45:15,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +14: [2023-04-29 09:45:15,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +17: [2023-04-29 09:45:16,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. + 2: [2023-04-29 09:45:16,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:16,000] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 + 7: [2023-04-29 09:45:16,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:16,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +57: [2023-04-29 09:45:16,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_02_optim_states.pt. +57: [2023-04-29 09:45:16,002] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 458 + 7: [2023-04-29 09:45:16,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:16,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:16,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_02_optim_states.pt. +57: [2023-04-29 09:45:16,003] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 462 +13: [2023-04-29 09:45:16,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +27: [2023-04-29 09:45:16,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +47: [2023-04-29 09:45:16,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_02_optim_states.pt. +47: [2023-04-29 09:45:16,004] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 376 + 7: [2023-04-29 09:45:16,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +14: [2023-04-29 09:45:16,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +30: [2023-04-29 09:45:16,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +30: [2023-04-29 09:45:16,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +30: [2023-04-29 09:45:16,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +30: [2023-04-29 09:45:16,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +21: [2023-04-29 09:45:16,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +28: [2023-04-29 09:45:16,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:16,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:16,007] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 +62: [2023-04-29 09:45:16,007] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 502 +21: [2023-04-29 09:45:16,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:16,009] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 458 +21: [2023-04-29 09:45:16,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +13: [2023-04-29 09:45:16,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +57: [2023-04-29 09:45:16,011] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 462 +47: [2023-04-29 09:45:16,011] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 376 +28: [2023-04-29 09:45:16,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:16,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +38: [2023-04-29 09:45:16,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +38: [2023-04-29 09:45:16,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +38: [2023-04-29 09:45:16,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +28: [2023-04-29 09:45:16,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 8: [2023-04-29 09:45:16,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +13: [2023-04-29 09:45:16,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +28: [2023-04-29 09:45:16,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +47: [2023-04-29 09:45:16,017] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 378 +27: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +27: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +58: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +29: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-04-29 09:45:16,019] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 +27: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +28: [2023-04-29 09:45:16,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +47: [2023-04-29 09:45:16,020] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 382 + 8: [2023-04-29 09:45:16,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +27: [2023-04-29 09:45:16,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +41: [2023-04-29 09:45:16,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +13: [2023-04-29 09:45:16,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +42: [2023-04-29 09:45:16,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +13: [2023-04-29 09:45:16,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +58: [2023-04-29 09:45:16,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +23: [2023-04-29 09:45:16,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:16,025] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 +28: [2023-04-29 09:45:16,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 2: [2023-04-29 09:45:16,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 2: [2023-04-29 09:45:16,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-04-29 09:45:16,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... + 2: [2023-04-29 09:45:16,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +40: [2023-04-29 09:45:16,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_02_optim_states.pt. +22: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +22: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +40: [2023-04-29 09:45:16,026] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 320 +16: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +15: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +41: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +16: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +15: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +42: [2023-04-29 09:45:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:16,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +17: [2023-04-29 09:45:16,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +22: [2023-04-29 09:45:16,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +17: [2023-04-29 09:45:16,027] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 +16: [2023-04-29 09:45:16,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +16: [2023-04-29 09:45:16,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +23: [2023-04-29 09:45:16,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +41: [2023-04-29 09:45:16,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +13: [2023-04-29 09:45:16,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +62: [2023-04-29 09:45:16,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_02_optim_states.pt. +62: [2023-04-29 09:45:16,030] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 500 +41: [2023-04-29 09:45:16,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 8: [2023-04-29 09:45:16,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:16,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 8: [2023-04-29 09:45:16,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +42: [2023-04-29 09:45:16,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +42: [2023-04-29 09:45:16,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:16,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +50: [2023-04-29 09:45:16,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +50: [2023-04-29 09:45:16,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +17: [2023-04-29 09:45:16,034] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 +50: [2023-04-29 09:45:16,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +40: [2023-04-29 09:45:16,034] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 320 +41: [2023-04-29 09:45:16,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +47: [2023-04-29 09:45:16,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_02_optim_states.pt. +47: [2023-04-29 09:45:16,035] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 380 +41: [2023-04-29 09:45:16,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +58: [2023-04-29 09:45:16,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +58: [2023-04-29 09:45:16,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:16,036] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 500 + 8: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 8: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +15: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +15: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... + 8: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +23: [2023-04-29 09:45:16,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +58: [2023-04-29 09:45:16,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +41: [2023-04-29 09:45:16,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +38: [2023-04-29 09:45:16,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +58: [2023-04-29 09:45:16,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +47: [2023-04-29 09:45:16,042] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 380 +23: [2023-04-29 09:45:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:16,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +38: [2023-04-29 09:45:16,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +58: [2023-04-29 09:45:16,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +23: [2023-04-29 09:45:16,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:16,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +23: [2023-04-29 09:45:16,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +58: [2023-04-29 09:45:16,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +39: [2023-04-29 09:45:16,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +39: [2023-04-29 09:45:16,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +39: [2023-04-29 09:45:16,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +39: [2023-04-29 09:45:16,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:16,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +62: [2023-04-29 09:45:16,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_02_optim_states.pt. +62: [2023-04-29 09:45:16,046] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 496 +26: [2023-04-29 09:45:16,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +26: [2023-04-29 09:45:16,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +26: [2023-04-29 09:45:16,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +26: [2023-04-29 09:45:16,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +27: [2023-04-29 09:45:16,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:16,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +23: [2023-04-29 09:45:16,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +23: [2023-04-29 09:45:16,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 6: [2023-04-29 09:45:16,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... +31: [2023-04-29 09:45:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +31: [2023-04-29 09:45:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-04-29 09:45:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +22: [2023-04-29 09:45:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +62: [2023-04-29 09:45:16,053] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 496 +30: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +30: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +22: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:16,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +22: [2023-04-29 09:45:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +22: [2023-04-29 09:45:16,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 6: [2023-04-29 09:45:16,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +30: [2023-04-29 09:45:16,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +30: [2023-04-29 09:45:16,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 6: [2023-04-29 09:45:16,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 6: [2023-04-29 09:45:16,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:16,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +59: [2023-04-29 09:45:16,058] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 474 +15: [2023-04-29 09:45:16,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 2: [2023-04-29 09:45:16,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:16,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +50: [2023-04-29 09:45:16,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +27: [2023-04-29 09:45:16,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 3: [2023-04-29 09:45:16,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +46: [2023-04-29 09:45:16,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_02_optim_states.pt. +46: [2023-04-29 09:45:16,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 372 +61: [2023-04-29 09:45:16,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_02_optim_states.pt. + 6: [2023-04-29 09:45:16,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +61: [2023-04-29 09:45:16,066] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 494 + 2: [2023-04-29 09:45:16,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 2: [2023-04-29 09:45:16,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +46: [2023-04-29 09:45:16,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_02_optim_states.pt. +46: [2023-04-29 09:45:16,068] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 368 + 6: [2023-04-29 09:45:16,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:16,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +29: [2023-04-29 09:45:16,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +29: [2023-04-29 09:45:16,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:16,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:16,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +50: [2023-04-29 09:45:16,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:16,071] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 472 +17: [2023-04-29 09:45:16,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +17: [2023-04-29 09:45:16,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +50: [2023-04-29 09:45:16,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:16,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:16,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +17: [2023-04-29 09:45:16,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +39: [2023-04-29 09:45:16,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +39: [2023-04-29 09:45:16,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +39: [2023-04-29 09:45:16,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +46: [2023-04-29 09:45:16,072] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 372 + 7: [2023-04-29 09:45:16,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-04-29 09:45:16,072] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 +39: [2023-04-29 09:45:16,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +59: [2023-04-29 09:45:16,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 476 +61: [2023-04-29 09:45:16,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 494 +21: [2023-04-29 09:45:16,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +52: [2023-04-29 09:45:16,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_02_optim_states.pt. +52: [2023-04-29 09:45:16,074] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 420 +46: [2023-04-29 09:45:16,075] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 368 +43: [2023-04-29 09:45:16,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +43: [2023-04-29 09:45:16,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +43: [2023-04-29 09:45:16,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +43: [2023-04-29 09:45:16,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +26: [2023-04-29 09:45:16,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:16,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 2: [2023-04-29 09:45:16,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 2: [2023-04-29 09:45:16,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 7: [2023-04-29 09:45:16,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:16,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:16,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +54: [2023-04-29 09:45:16,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +54: [2023-04-29 09:45:16,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +42: [2023-04-29 09:45:16,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:16,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_02_optim_states.pt. +33: [2023-04-29 09:45:16,078] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 264 + 7: [2023-04-29 09:45:16,079] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 +26: [2023-04-29 09:45:16,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:16,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:16,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +32: [2023-04-29 09:45:16,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +21: [2023-04-29 09:45:16,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:16,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +24: [2023-04-29 09:45:16,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +52: [2023-04-29 09:45:16,081] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 420 +24: [2023-04-29 09:45:16,081] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 +42: [2023-04-29 09:45:16,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +59: [2023-04-29 09:45:16,081] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 478 +21: [2023-04-29 09:45:16,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +16: [2023-04-29 09:45:16,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +16: [2023-04-29 09:45:16,082] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 +21: [2023-04-29 09:45:16,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +54: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +25: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +25: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 7: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +30: [2023-04-29 09:45:16,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +32: [2023-04-29 09:45:16,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +33: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_02_optim_states.pt. +32: [2023-04-29 09:45:16,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,083] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 268 +32: [2023-04-29 09:45:16,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +25: [2023-04-29 09:45:16,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +33: [2023-04-29 09:45:16,085] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 264 + 7: [2023-04-29 09:45:16,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:16,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:16,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:16,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +30: [2023-04-29 09:45:16,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +24: [2023-04-29 09:45:16,087] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 +21: [2023-04-29 09:45:16,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 7: [2023-04-29 09:45:16,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:16,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 + 7: [2023-04-29 09:45:16,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +42: [2023-04-29 09:45:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 5: [2023-04-29 09:45:16,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 5: [2023-04-29 09:45:16,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 9: [2023-04-29 09:45:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +46: [2023-04-29 09:45:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_02_optim_states.pt. + 9: [2023-04-29 09:45:16,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +46: [2023-04-29 09:45:16,093] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 370 +38: [2023-04-29 09:45:16,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +16: [2023-04-29 09:45:16,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +42: [2023-04-29 09:45:16,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +28: [2023-04-29 09:45:16,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +28: [2023-04-29 09:45:16,096] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 +27: [2023-04-29 09:45:16,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +27: [2023-04-29 09:45:16,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +27: [2023-04-29 09:45:16,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-04-29 09:45:16,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... + 7: [2023-04-29 09:45:16,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +57: [2023-04-29 09:45:16,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_02_optim_states.pt. +17: [2023-04-29 09:45:16,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:16,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:16,098] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 456 +16: [2023-04-29 09:45:16,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +38: [2023-04-29 09:45:16,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 7: [2023-04-29 09:45:16,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +33: [2023-04-29 09:45:16,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 268 +29: [2023-04-29 09:45:16,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:16,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +46: [2023-04-29 09:45:16,100] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 370 +43: [2023-04-29 09:45:16,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +29: [2023-04-29 09:45:16,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +52: [2023-04-29 09:45:16,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_02_optim_states.pt. +52: [2023-04-29 09:45:16,102] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 422 +29: [2023-04-29 09:45:16,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +42: [2023-04-29 09:45:16,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +42: [2023-04-29 09:45:16,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +16: [2023-04-29 09:45:16,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +52: [2023-04-29 09:45:16,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_02_optim_states.pt. +52: [2023-04-29 09:45:16,103] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 418 +16: [2023-04-29 09:45:16,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +28: [2023-04-29 09:45:16,105] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 +57: [2023-04-29 09:45:16,105] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 456 +32: [2023-04-29 09:45:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +29: [2023-04-29 09:45:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +16: [2023-04-29 09:45:16,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +16: [2023-04-29 09:45:16,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +43: [2023-04-29 09:45:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +22: [2023-04-29 09:45:16,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +43: [2023-04-29 09:45:16,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +43: [2023-04-29 09:45:16,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +17: [2023-04-29 09:45:16,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +42: [2023-04-29 09:45:16,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +52: [2023-04-29 09:45:16,108] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 422 +42: [2023-04-29 09:45:16,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +38: [2023-04-29 09:45:16,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +38: [2023-04-29 09:45:16,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +32: [2023-04-29 09:45:16,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +32: [2023-04-29 09:45:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:16,111] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 418 +24: [2023-04-29 09:45:16,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-04-29 09:45:16,111] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 +22: [2023-04-29 09:45:16,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +25: [2023-04-29 09:45:16,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +38: [2023-04-29 09:45:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +16: [2023-04-29 09:45:16,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +54: [2023-04-29 09:45:16,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +16: [2023-04-29 09:45:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 3: [2023-04-29 09:45:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +52: [2023-04-29 09:45:16,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_02_optim_states.pt. +49: [2023-04-29 09:45:16,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_02_optim_states.pt. +52: [2023-04-29 09:45:16,113] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 416 +49: [2023-04-29 09:45:16,114] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 394 +54: [2023-04-29 09:45:16,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +38: [2023-04-29 09:45:16,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +38: [2023-04-29 09:45:16,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +54: [2023-04-29 09:45:16,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:16,115] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 282 + 3: [2023-04-29 09:45:16,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +38: [2023-04-29 09:45:16,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +27: [2023-04-29 09:45:16,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:16,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +24: [2023-04-29 09:45:16,118] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 + 9: [2023-04-29 09:45:16,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:16,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +25: [2023-04-29 09:45:16,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +52: [2023-04-29 09:45:16,120] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 416 +35: [2023-04-29 09:45:16,120] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 280 +35: [2023-04-29 09:45:16,121] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 284 +49: [2023-04-29 09:45:16,122] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 394 +29: [2023-04-29 09:45:16,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +29: [2023-04-29 09:45:16,122] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 +25: [2023-04-29 09:45:16,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +15: [2023-04-29 09:45:16,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +15: [2023-04-29 09:45:16,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +19: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +19: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +22: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +19: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +19: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:16,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 9: [2023-04-29 09:45:16,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +22: [2023-04-29 09:45:16,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +22: [2023-04-29 09:45:16,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:16,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_02_optim_states.pt. +54: [2023-04-29 09:45:16,129] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 434 +22: [2023-04-29 09:45:16,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:16,130] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 +55: [2023-04-29 09:45:16,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_02_optim_states.pt. +50: [2023-04-29 09:45:16,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +55: [2023-04-29 09:45:16,132] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 444 +39: [2023-04-29 09:45:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +15: [2023-04-29 09:45:16,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 1: [2023-04-29 09:45:16,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +33: [2023-04-29 09:45:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +15: [2023-04-29 09:45:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +37: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +37: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +18: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +18: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 1: [2023-04-29 09:45:16,132] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 +18: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +37: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +18: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +37: [2023-04-29 09:45:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +12: [2023-04-29 09:45:16,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +22: [2023-04-29 09:45:16,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +12: [2023-04-29 09:45:16,134] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 +22: [2023-04-29 09:45:16,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +46: [2023-04-29 09:45:16,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_02_optim_states.pt. +46: [2023-04-29 09:45:16,135] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 374 +27: [2023-04-29 09:45:16,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:16,137] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 434 + 4: [2023-04-29 09:45:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +39: [2023-04-29 09:45:16,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 4: [2023-04-29 09:45:16,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 +54: [2023-04-29 09:45:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_02_optim_states.pt. +54: [2023-04-29 09:45:16,138] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 436 +28: [2023-04-29 09:45:16,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +28: [2023-04-29 09:45:16,138] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +55: [2023-04-29 09:45:16,139] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 444 +39: [2023-04-29 09:45:16,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +39: [2023-04-29 09:45:16,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 3: [2023-04-29 09:45:16,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 1: [2023-04-29 09:45:16,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +12: [2023-04-29 09:45:16,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 +15: [2023-04-29 09:45:16,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +15: [2023-04-29 09:45:16,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +46: [2023-04-29 09:45:16,141] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 374 +15: [2023-04-29 09:45:16,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +15: [2023-04-29 09:45:16,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +39: [2023-04-29 09:45:16,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +50: [2023-04-29 09:45:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +50: [2023-04-29 09:45:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 4: [2023-04-29 09:45:16,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 4: [2023-04-29 09:45:16,144] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 + 4: [2023-04-29 09:45:16,144] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 + 4: [2023-04-29 09:45:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +28: [2023-04-29 09:45:16,144] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 + 4: [2023-04-29 09:45:16,144] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 +39: [2023-04-29 09:45:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +39: [2023-04-29 09:45:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +50: [2023-04-29 09:45:16,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +26: [2023-04-29 09:45:16,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:16,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +39: [2023-04-29 09:45:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +54: [2023-04-29 09:45:16,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 436 + 7: [2023-04-29 09:45:16,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +19: [2023-04-29 09:45:16,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 7: [2023-04-29 09:45:16,148] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 +26: [2023-04-29 09:45:16,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:16,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +27: [2023-04-29 09:45:16,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 4: [2023-04-29 09:45:16,150] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 +50: [2023-04-29 09:45:16,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +55: [2023-04-29 09:45:16,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_02_optim_states.pt. +55: [2023-04-29 09:45:16,151] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 442 +50: [2023-04-29 09:45:16,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 5: [2023-04-29 09:45:16,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:16,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +27: [2023-04-29 09:45:16,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 4: [2023-04-29 09:45:16,152] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +26: [2023-04-29 09:45:16,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +50: [2023-04-29 09:45:16,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +30: [2023-04-29 09:45:16,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +26: [2023-04-29 09:45:16,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:16,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +24: [2023-04-29 09:45:16,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. +26: [2023-04-29 09:45:16,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +54: [2023-04-29 09:45:16,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_02_optim_states.pt. +24: [2023-04-29 09:45:16,156] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 +54: [2023-04-29 09:45:16,156] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 438 +19: [2023-04-29 09:45:16,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +26: [2023-04-29 09:45:16,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:16,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +30: [2023-04-29 09:45:16,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +55: [2023-04-29 09:45:16,158] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 442 +55: [2023-04-29 09:45:16,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_02_optim_states.pt. + 7: [2023-04-29 09:45:16,159] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 +55: [2023-04-29 09:45:16,159] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 440 + 3: [2023-04-29 09:45:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-04-29 09:45:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... + 3: [2023-04-29 09:45:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... + 3: [2023-04-29 09:45:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +40: [2023-04-29 09:45:16,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +40: [2023-04-29 09:45:16,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +40: [2023-04-29 09:45:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +40: [2023-04-29 09:45:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +32: [2023-04-29 09:45:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +17: [2023-04-29 09:45:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +17: [2023-04-29 09:45:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +37: [2023-04-29 09:45:16,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:16,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_02_optim_states.pt. +18: [2023-04-29 09:45:16,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +50: [2023-04-29 09:45:16,162] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 404 +30: [2023-04-29 09:45:16,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 5: [2023-04-29 09:45:16,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +26: [2023-04-29 09:45:16,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +37: [2023-04-29 09:45:16,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:16,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +24: [2023-04-29 09:45:16,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +43: [2023-04-29 09:45:16,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +30: [2023-04-29 09:45:16,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +54: [2023-04-29 09:45:16,164] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 438 +33: [2023-04-29 09:45:16,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +26: [2023-04-29 09:45:16,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +30: [2023-04-29 09:45:16,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +33: [2023-04-29 09:45:16,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +28: [2023-04-29 09:45:16,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +32: [2023-04-29 09:45:16,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +28: [2023-04-29 09:45:16,165] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +33: [2023-04-29 09:45:16,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_02_optim_states.pt. +37: [2023-04-29 09:45:16,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:16,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +33: [2023-04-29 09:45:16,166] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 270 +55: [2023-04-29 09:45:16,166] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 440 + 5: [2023-04-29 09:45:16,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 5: [2023-04-29 09:45:16,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +51: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +51: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +51: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +51: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +43: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 5: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +34: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +34: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +34: [2023-04-29 09:45:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +18: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +34: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +18: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +17: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +17: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +48: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +48: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +30: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +62: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +62: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +62: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +33: [2023-04-29 09:45:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +43: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +16: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. +16: [2023-04-29 09:45:16,170] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 +25: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +48: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +48: [2023-04-29 09:45:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +50: [2023-04-29 09:45:16,172] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 404 +35: [2023-04-29 09:45:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +35: [2023-04-29 09:45:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +35: [2023-04-29 09:45:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +29: [2023-04-29 09:45:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +35: [2023-04-29 09:45:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +17: [2023-04-29 09:45:16,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:16,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 1: [2023-04-29 09:45:16,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +33: [2023-04-29 09:45:16,174] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 270 + 1: [2023-04-29 09:45:16,174] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +28: [2023-04-29 09:45:16,175] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +43: [2023-04-29 09:45:16,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +16: [2023-04-29 09:45:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-04-29 09:45:16,175] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 +28: [2023-04-29 09:45:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-04-29 09:45:16,176] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +25: [2023-04-29 09:45:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +17: [2023-04-29 09:45:16,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +29: [2023-04-29 09:45:16,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +14: [2023-04-29 09:45:16,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +16: [2023-04-29 09:45:16,178] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +14: [2023-04-29 09:45:16,178] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 +32: [2023-04-29 09:45:16,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:16,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:16,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +56: [2023-04-29 09:45:16,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +56: [2023-04-29 09:45:16,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +56: [2023-04-29 09:45:16,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +55: [2023-04-29 09:45:16,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_02_optim_states.pt. +55: [2023-04-29 09:45:16,180] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 446 +17: [2023-04-29 09:45:16,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:16,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:16,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:16,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +32: [2023-04-29 09:45:16,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 1: [2023-04-29 09:45:16,182] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +28: [2023-04-29 09:45:16,182] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 + 3: [2023-04-29 09:45:16,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +43: [2023-04-29 09:45:16,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:16,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +22: [2023-04-29 09:45:16,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +22: [2023-04-29 09:45:16,183] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +32: [2023-04-29 09:45:16,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +16: [2023-04-29 09:45:16,184] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +29: [2023-04-29 09:45:16,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +20: [2023-04-29 09:45:16,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +20: [2023-04-29 09:45:16,185] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 + 3: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +45: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +45: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +45: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +45: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +32: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +40: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_02_optim_states.pt. +17: [2023-04-29 09:45:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +54: [2023-04-29 09:45:16,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +14: [2023-04-29 09:45:16,186] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 +40: [2023-04-29 09:45:16,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 324 +32: [2023-04-29 09:45:16,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +55: [2023-04-29 09:45:16,188] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 446 + 9: [2023-04-29 09:45:16,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +20: [2023-04-29 09:45:16,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +24: [2023-04-29 09:45:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +20: [2023-04-29 09:45:16,189] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +40: [2023-04-29 09:45:16,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +24: [2023-04-29 09:45:16,189] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 +54: [2023-04-29 09:45:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +43: [2023-04-29 09:45:16,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +20: [2023-04-29 09:45:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-04-29 09:45:16,189] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +43: [2023-04-29 09:45:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +16: [2023-04-29 09:45:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +16: [2023-04-29 09:45:16,190] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 +22: [2023-04-29 09:45:16,191] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 +20: [2023-04-29 09:45:16,191] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 +54: [2023-04-29 09:45:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +55: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +55: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +40: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +29: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +12: [2023-04-29 09:45:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-04-29 09:45:16,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +29: [2023-04-29 09:45:16,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 + 9: [2023-04-29 09:45:16,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:16,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-04-29 09:45:16,194] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +54: [2023-04-29 09:45:16,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +34: [2023-04-29 09:45:16,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +40: [2023-04-29 09:45:16,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +40: [2023-04-29 09:45:16,196] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 324 +34: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +34: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +51: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 9: [2023-04-29 09:45:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +24: [2023-04-29 09:45:16,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 +20: [2023-04-29 09:45:16,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 +54: [2023-04-29 09:45:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +20: [2023-04-29 09:45:16,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 +16: [2023-04-29 09:45:16,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 + 9: [2023-04-29 09:45:16,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 9: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +34: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +54: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +48: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_02_optim_states.pt. +35: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:16,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 286 +48: [2023-04-29 09:45:16,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +48: [2023-04-29 09:45:16,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:16,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +12: [2023-04-29 09:45:16,200] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 + 9: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +29: [2023-04-29 09:45:16,201] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +12: [2023-04-29 09:45:16,201] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +63: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +51: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +63: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +35: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:16,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:16,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +54: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +51: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +51: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +47: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +47: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +25: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +47: [2023-04-29 09:45:16,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +47: [2023-04-29 09:45:16,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +62: [2023-04-29 09:45:16,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +57: [2023-04-29 09:45:16,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +57: [2023-04-29 09:45:16,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +57: [2023-04-29 09:45:16,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +57: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +56: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +62: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 7: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. + 9: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 7: [2023-04-29 09:45:16,206] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 + 3: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +56: [2023-04-29 09:45:16,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +25: [2023-04-29 09:45:16,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +14: [2023-04-29 09:45:16,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +14: [2023-04-29 09:45:16,208] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +35: [2023-04-29 09:45:16,208] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 286 +25: [2023-04-29 09:45:16,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +20: [2023-04-29 09:45:16,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-04-29 09:45:16,210] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 +56: [2023-04-29 09:45:16,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +45: [2023-04-29 09:45:16,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 1: [2023-04-29 09:45:16,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +45: [2023-04-29 09:45:16,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 1: [2023-04-29 09:45:16,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 +19: [2023-04-29 09:45:16,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 1: [2023-04-29 09:45:16,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. + 7: [2023-04-29 09:45:16,215] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +45: [2023-04-29 09:45:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +45: [2023-04-29 09:45:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +20: [2023-04-29 09:45:16,217] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 +19: [2023-04-29 09:45:16,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:16,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +14: [2023-04-29 09:45:16,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 1: [2023-04-29 09:45:16,215] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 + 1: [2023-04-29 09:45:16,217] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 +14: [2023-04-29 09:45:16,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 +14: [2023-04-29 09:45:16,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +14: [2023-04-29 09:45:16,217] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +14: [2023-04-29 09:45:16,218] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +19: [2023-04-29 09:45:16,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +37: [2023-04-29 09:45:16,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:16,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +19: [2023-04-29 09:45:16,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +19: [2023-04-29 09:45:16,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +55: [2023-04-29 09:45:16,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:16,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +55: [2023-04-29 09:45:16,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:16,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +21: [2023-04-29 09:45:16,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. +21: [2023-04-29 09:45:16,229] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 +63: [2023-04-29 09:45:16,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:16,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_02_optim_states.pt. +63: [2023-04-29 09:45:16,230] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 506 +14: [2023-04-29 09:45:16,222] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 + 1: [2023-04-29 09:45:16,223] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +14: [2023-04-29 09:45:16,224] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 +37: [2023-04-29 09:45:16,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +18: [2023-04-29 09:45:16,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +57: [2023-04-29 09:45:16,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +19: [2023-04-29 09:45:16,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +19: [2023-04-29 09:45:16,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +63: [2023-04-29 09:45:16,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +12: [2023-04-29 09:45:16,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +63: [2023-04-29 09:45:16,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:16,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +12: [2023-04-29 09:45:16,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 +10: [2023-04-29 09:45:16,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 +37: [2023-04-29 09:45:16,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:16,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +47: [2023-04-29 09:45:16,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +57: [2023-04-29 09:45:16,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +57: [2023-04-29 09:45:16,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +47: [2023-04-29 09:45:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +63: [2023-04-29 09:45:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +18: [2023-04-29 09:45:16,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +21: [2023-04-29 09:45:16,236] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 +18: [2023-04-29 09:45:16,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +37: [2023-04-29 09:45:16,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +46: [2023-04-29 09:45:16,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +46: [2023-04-29 09:45:16,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +63: [2023-04-29 09:45:16,238] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 506 +46: [2023-04-29 09:45:16,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:16,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:16,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +12: [2023-04-29 09:45:16,240] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 +37: [2023-04-29 09:45:16,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +10: [2023-04-29 09:45:16,240] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +18: [2023-04-29 09:45:16,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +18: [2023-04-29 09:45:16,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +33: [2023-04-29 09:45:16,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:16,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +33: [2023-04-29 09:45:16,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:16,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +18: [2023-04-29 09:45:16,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +33: [2023-04-29 09:45:16,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:16,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +44: [2023-04-29 09:45:16,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +44: [2023-04-29 09:45:16,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +44: [2023-04-29 09:45:16,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +33: [2023-04-29 09:45:16,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +18: [2023-04-29 09:45:16,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +40: [2023-04-29 09:45:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:16,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +48: [2023-04-29 09:45:16,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +18: [2023-04-29 09:45:16,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +34: [2023-04-29 09:45:16,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:16,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. + 7: [2023-04-29 09:45:16,258] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 +40: [2023-04-29 09:45:16,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +40: [2023-04-29 09:45:16,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:16,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +56: [2023-04-29 09:45:16,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:16,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:16,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +51: [2023-04-29 09:45:16,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:16,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:16,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:16,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +40: [2023-04-29 09:45:16,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:16,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:16,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:16,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +40: [2023-04-29 09:45:16,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +35: [2023-04-29 09:45:16,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +35: [2023-04-29 09:45:16,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +56: [2023-04-29 09:45:16,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +46: [2023-04-29 09:45:16,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +48: [2023-04-29 09:45:16,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +31: [2023-04-29 09:45:16,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +31: [2023-04-29 09:45:16,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 7: [2023-04-29 09:45:16,267] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 +34: [2023-04-29 09:45:16,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +34: [2023-04-29 09:45:16,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +51: [2023-04-29 09:45:16,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +31: [2023-04-29 09:45:16,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +40: [2023-04-29 09:45:16,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +40: [2023-04-29 09:45:16,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +31: [2023-04-29 09:45:16,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +62: [2023-04-29 09:45:16,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +34: [2023-04-29 09:45:16,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +21: [2023-04-29 09:45:16,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +21: [2023-04-29 09:45:16,270] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 +25: [2023-04-29 09:45:16,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +25: [2023-04-29 09:45:16,271] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 +56: [2023-04-29 09:45:16,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:16,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +51: [2023-04-29 09:45:16,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 6: [2023-04-29 09:45:16,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. + 6: [2023-04-29 09:45:16,273] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +56: [2023-04-29 09:45:16,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:16,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +35: [2023-04-29 09:45:16,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +35: [2023-04-29 09:45:16,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:16,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_22-model_01-model_states.pt. +45: [2023-04-29 09:45:16,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +45: [2023-04-29 09:45:16,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:16,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:16,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +44: [2023-04-29 09:45:16,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +21: [2023-04-29 09:45:16,277] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 +51: [2023-04-29 09:45:16,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:16,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +44: [2023-04-29 09:45:16,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +56: [2023-04-29 09:45:16,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +25: [2023-04-29 09:45:16,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +51: [2023-04-29 09:45:16,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +35: [2023-04-29 09:45:16,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 6: [2023-04-29 09:45:16,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 +45: [2023-04-29 09:45:16,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +45: [2023-04-29 09:45:16,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +51: [2023-04-29 09:45:16,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:16,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +48: [2023-04-29 09:45:16,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +35: [2023-04-29 09:45:16,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +62: [2023-04-29 09:45:16,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +22: [2023-04-29 09:45:16,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +62: [2023-04-29 09:45:16,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +62: [2023-04-29 09:45:16,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +22: [2023-04-29 09:45:16,283] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +45: [2023-04-29 09:45:16,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +51: [2023-04-29 09:45:16,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +25: [2023-04-29 09:45:16,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +25: [2023-04-29 09:45:16,285] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +45: [2023-04-29 09:45:16,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +56: [2023-04-29 09:45:16,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +51: [2023-04-29 09:45:16,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +22: [2023-04-29 09:45:16,290] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 + 8: [2023-04-29 09:45:16,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +47: [2023-04-29 09:45:16,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 8: [2023-04-29 09:45:16,290] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 +45: [2023-04-29 09:45:16,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +62: [2023-04-29 09:45:16,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +62: [2023-04-29 09:45:16,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +25: [2023-04-29 09:45:16,291] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +25: [2023-04-29 09:45:16,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +45: [2023-04-29 09:45:16,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +25: [2023-04-29 09:45:16,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 +57: [2023-04-29 09:45:16,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 6: [2023-04-29 09:45:16,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +47: [2023-04-29 09:45:16,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 6: [2023-04-29 09:45:16,295] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 +63: [2023-04-29 09:45:16,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:16,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 8: [2023-04-29 09:45:16,297] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 +57: [2023-04-29 09:45:16,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +25: [2023-04-29 09:45:16,298] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 +31: [2023-04-29 09:45:16,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:16,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:16,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +31: [2023-04-29 09:45:16,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +55: [2023-04-29 09:45:16,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:16,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:16,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 6: [2023-04-29 09:45:16,302] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 +57: [2023-04-29 09:45:16,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:16,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:16,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:16,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +55: [2023-04-29 09:45:16,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +63: [2023-04-29 09:45:16,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +21: [2023-04-29 09:45:16,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. + 0: [2023-04-29 09:45:16,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +21: [2023-04-29 09:45:16,305] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 + 0: [2023-04-29 09:45:16,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +57: [2023-04-29 09:45:16,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 0: [2023-04-29 09:45:16,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... + 0: [2023-04-29 09:45:16,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt... +55: [2023-04-29 09:45:16,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +55: [2023-04-29 09:45:16,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +55: [2023-04-29 09:45:16,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +63: [2023-04-29 09:45:16,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +57: [2023-04-29 09:45:16,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +10: [2023-04-29 09:45:16,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-04-29 09:45:16,309] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 +57: [2023-04-29 09:45:16,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:16,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +47: [2023-04-29 09:45:16,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +47: [2023-04-29 09:45:16,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +47: [2023-04-29 09:45:16,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +57: [2023-04-29 09:45:16,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +63: [2023-04-29 09:45:16,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +63: [2023-04-29 09:45:16,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +21: [2023-04-29 09:45:16,312] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +57: [2023-04-29 09:45:16,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +23: [2023-04-29 09:45:16,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +23: [2023-04-29 09:45:16,315] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 +10: [2023-04-29 09:45:16,315] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 +63: [2023-04-29 09:45:16,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +63: [2023-04-29 09:45:16,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_02_optim_states.pt. +63: [2023-04-29 09:45:16,317] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 504 +63: [2023-04-29 09:45:16,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +13: [2023-04-29 09:45:16,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +13: [2023-04-29 09:45:16,319] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +11: [2023-04-29 09:45:16,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-04-29 09:45:16,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 +13: [2023-04-29 09:45:16,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +13: [2023-04-29 09:45:16,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 +23: [2023-04-29 09:45:16,322] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 +10: [2023-04-29 09:45:16,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +10: [2023-04-29 09:45:16,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 +63: [2023-04-29 09:45:16,326] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 504 +13: [2023-04-29 09:45:16,327] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 +11: [2023-04-29 09:45:16,328] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 +13: [2023-04-29 09:45:16,328] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 +21: [2023-04-29 09:45:16,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +21: [2023-04-29 09:45:16,330] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 +10: [2023-04-29 09:45:16,333] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 +46: [2023-04-29 09:45:16,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:16,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +10: [2023-04-29 09:45:16,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-04-29 09:45:16,335] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +44: [2023-04-29 09:45:16,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +21: [2023-04-29 09:45:16,338] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 +46: [2023-04-29 09:45:16,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:16,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +46: [2023-04-29 09:45:16,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +13: [2023-04-29 09:45:16,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +44: [2023-04-29 09:45:16,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +13: [2023-04-29 09:45:16,342] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 +10: [2023-04-29 09:45:16,343] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +46: [2023-04-29 09:45:16,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +44: [2023-04-29 09:45:16,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +13: [2023-04-29 09:45:16,349] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 +44: [2023-04-29 09:45:16,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:16,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +44: [2023-04-29 09:45:16,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +22: [2023-04-29 09:45:16,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-04-29 09:45:16,353] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 +44: [2023-04-29 09:45:16,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +44: [2023-04-29 09:45:16,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 6: [2023-04-29 09:45:16,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-04-29 09:45:16,356] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 +22: [2023-04-29 09:45:16,360] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 + 6: [2023-04-29 09:45:16,363] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 +26: [2023-04-29 09:45:16,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +26: [2023-04-29 09:45:16,364] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 +31: [2023-04-29 09:45:16,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +23: [2023-04-29 09:45:16,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +23: [2023-04-29 09:45:16,369] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +26: [2023-04-29 09:45:16,370] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +11: [2023-04-29 09:45:16,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +11: [2023-04-29 09:45:16,371] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 +31: [2023-04-29 09:45:16,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:16,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:16,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:16,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:16,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +13: [2023-04-29 09:45:16,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-04-29 09:45:16,376] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 +23: [2023-04-29 09:45:16,377] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 + 0: [2023-04-29 09:45:16,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. + 0: [2023-04-29 09:45:16,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:16,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 0: [2023-04-29 09:45:16,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_00-model_states.pt. +31: [2023-04-29 09:45:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +31: [2023-04-29 09:45:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +11: [2023-04-29 09:45:16,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 + 0: [2023-04-29 09:45:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 0: [2023-04-29 09:45:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +13: [2023-04-29 09:45:16,384] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 +11: [2023-04-29 09:45:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +11: [2023-04-29 09:45:16,387] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 + 4: [2023-04-29 09:45:16,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. + 4: [2023-04-29 09:45:16,389] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 + 8: [2023-04-29 09:45:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. + 8: [2023-04-29 09:45:16,390] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 +41: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +41: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... + 0: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt... +59: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +59: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +59: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +59: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +41: [2023-04-29 09:45:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +11: [2023-04-29 09:45:16,394] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 + 0: [2023-04-29 09:45:16,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-04-29 09:45:16,395] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 +60: [2023-04-29 09:45:16,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_02_optim_states.pt. +60: [2023-04-29 09:45:16,396] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 482 + 4: [2023-04-29 09:45:16,396] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 + 8: [2023-04-29 09:45:16,396] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 +62: [2023-04-29 09:45:16,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_02_optim_states.pt. +62: [2023-04-29 09:45:16,397] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 498 +23: [2023-04-29 09:45:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +23: [2023-04-29 09:45:16,398] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +60: [2023-04-29 09:45:16,403] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 482 +26: [2023-04-29 09:45:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-04-29 09:45:16,403] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 + 0: [2023-04-29 09:45:16,403] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 +62: [2023-04-29 09:45:16,403] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 498 + 9: [2023-04-29 09:45:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. + 9: [2023-04-29 09:45:16,404] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 +23: [2023-04-29 09:45:16,405] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 +33: [2023-04-29 09:45:16,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_02_optim_states.pt. +26: [2023-04-29 09:45:16,410] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 +33: [2023-04-29 09:45:16,411] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 266 + 9: [2023-04-29 09:45:16,411] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 +19: [2023-04-29 09:45:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-04-29 09:45:16,415] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 +59: [2023-04-29 09:45:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 9: [2023-04-29 09:45:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. + 9: [2023-04-29 09:45:16,416] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +33: [2023-04-29 09:45:16,417] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 266 +41: [2023-04-29 09:45:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:16,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:16,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:16,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:16,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:16,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +19: [2023-04-29 09:45:16,422] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 + 9: [2023-04-29 09:45:16,423] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 +41: [2023-04-29 09:45:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:16,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +41: [2023-04-29 09:45:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +41: [2023-04-29 09:45:16,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +23: [2023-04-29 09:45:16,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +23: [2023-04-29 09:45:16,429] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +26: [2023-04-29 09:45:16,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-04-29 09:45:16,430] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 + 5: [2023-04-29 09:45:16,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 5: [2023-04-29 09:45:16,432] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 +41: [2023-04-29 09:45:16,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +41: [2023-04-29 09:45:16,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +41: [2023-04-29 09:45:16,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +41: [2023-04-29 09:45:16,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +26: [2023-04-29 09:45:16,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +26: [2023-04-29 09:45:16,434] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +23: [2023-04-29 09:45:16,437] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 + 0: [2023-04-29 09:45:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. + 0: [2023-04-29 09:45:16,438] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 +26: [2023-04-29 09:45:16,439] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 + 5: [2023-04-29 09:45:16,439] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 +26: [2023-04-29 09:45:16,441] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 +42: [2023-04-29 09:45:16,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +42: [2023-04-29 09:45:16,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +42: [2023-04-29 09:45:16,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +42: [2023-04-29 09:45:16,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. + 0: [2023-04-29 09:45:16,445] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 + 0: [2023-04-29 09:45:16,446] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 +58: [2023-04-29 09:45:16,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +58: [2023-04-29 09:45:16,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +58: [2023-04-29 09:45:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +58: [2023-04-29 09:45:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 + 0: [2023-04-29 09:45:16,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 + 0: [2023-04-29 09:45:16,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. + 5: [2023-04-29 09:45:16,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 0: [2023-04-29 09:45:16,463] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 +18: [2023-04-29 09:45:16,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. + 5: [2023-04-29 09:45:16,463] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 +18: [2023-04-29 09:45:16,464] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +42: [2023-04-29 09:45:16,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +41: [2023-04-29 09:45:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_03_optim_states.pt... +41: [2023-04-29 09:45:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_03_optim_states.pt... +41: [2023-04-29 09:45:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_03_optim_states.pt... +41: [2023-04-29 09:45:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_03_optim_states.pt... +42: [2023-04-29 09:45:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... + 0: [2023-04-29 09:45:16,470] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 + 5: [2023-04-29 09:45:16,471] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +18: [2023-04-29 09:45:16,471] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +42: [2023-04-29 09:45:16,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:16,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:16,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:16,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +42: [2023-04-29 09:45:16,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +42: [2023-04-29 09:45:16,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:16,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +42: [2023-04-29 09:45:16,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +42: [2023-04-29 09:45:16,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +42: [2023-04-29 09:45:16,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +42: [2023-04-29 09:45:16,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +42: [2023-04-29 09:45:16,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +42: [2023-04-29 09:45:16,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +30: [2023-04-29 09:45:16,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +30: [2023-04-29 09:45:16,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 +58: [2023-04-29 09:45:16,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:16,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:16,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:16,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:16,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:16,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +58: [2023-04-29 09:45:16,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +58: [2023-04-29 09:45:16,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +58: [2023-04-29 09:45:16,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:16,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +58: [2023-04-29 09:45:16,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +58: [2023-04-29 09:45:16,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... + 2: [2023-04-29 09:45:16,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. + 2: [2023-04-29 09:45:16,481] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 +58: [2023-04-29 09:45:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +58: [2023-04-29 09:45:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +58: [2023-04-29 09:45:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +58: [2023-04-29 09:45:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +30: [2023-04-29 09:45:16,482] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +63: [2023-04-29 09:45:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_02_optim_states.pt. +63: [2023-04-29 09:45:16,483] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 510 +17: [2023-04-29 09:45:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. +17: [2023-04-29 09:45:16,482] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 +15: [2023-04-29 09:45:16,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +15: [2023-04-29 09:45:16,485] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 +59: [2023-04-29 09:45:16,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 9: [2023-04-29 09:45:16,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. + 9: [2023-04-29 09:45:16,486] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 +59: [2023-04-29 09:45:16,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 2: [2023-04-29 09:45:16,490] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 +17: [2023-04-29 09:45:16,490] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +63: [2023-04-29 09:45:16,490] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 510 +59: [2023-04-29 09:45:16,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +15: [2023-04-29 09:45:16,491] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 +18: [2023-04-29 09:45:16,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-04-29 09:45:16,492] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 +59: [2023-04-29 09:45:16,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 9: [2023-04-29 09:45:16,494] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 +59: [2023-04-29 09:45:16,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +59: [2023-04-29 09:45:16,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +27: [2023-04-29 09:45:16,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-04-29 09:45:16,498] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 +18: [2023-04-29 09:45:16,499] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 + 2: [2023-04-29 09:45:16,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 2: [2023-04-29 09:45:16,502] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 +59: [2023-04-29 09:45:16,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +59: [2023-04-29 09:45:16,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +15: [2023-04-29 09:45:16,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +15: [2023-04-29 09:45:16,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 +30: [2023-04-29 09:45:16,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-04-29 09:45:16,505] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +27: [2023-04-29 09:45:16,508] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +15: [2023-04-29 09:45:16,511] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 + 2: [2023-04-29 09:45:16,511] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +42: [2023-04-29 09:45:16,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_03_optim_states.pt... +42: [2023-04-29 09:45:16,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_03_optim_states.pt... +30: [2023-04-29 09:45:16,514] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +42: [2023-04-29 09:45:16,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_03_optim_states.pt... +42: [2023-04-29 09:45:16,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_03_optim_states.pt... +30: [2023-04-29 09:45:16,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-04-29 09:45:16,520] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 +19: [2023-04-29 09:45:16,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-04-29 09:45:16,521] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 +58: [2023-04-29 09:45:16,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_03_optim_states.pt... +58: [2023-04-29 09:45:16,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_03_optim_states.pt... +58: [2023-04-29 09:45:16,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_03_optim_states.pt... +58: [2023-04-29 09:45:16,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_03_optim_states.pt... +60: [2023-04-29 09:45:16,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +60: [2023-04-29 09:45:16,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +60: [2023-04-29 09:45:16,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +60: [2023-04-29 09:45:16,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +38: [2023-04-29 09:45:16,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +38: [2023-04-29 09:45:16,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +38: [2023-04-29 09:45:16,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +38: [2023-04-29 09:45:16,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +30: [2023-04-29 09:45:16,527] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 +39: [2023-04-29 09:45:16,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +39: [2023-04-29 09:45:16,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +39: [2023-04-29 09:45:16,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +39: [2023-04-29 09:45:16,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +19: [2023-04-29 09:45:16,529] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 +53: [2023-04-29 09:45:16,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +53: [2023-04-29 09:45:16,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +53: [2023-04-29 09:45:16,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +53: [2023-04-29 09:45:16,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +49: [2023-04-29 09:45:16,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +49: [2023-04-29 09:45:16,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,534] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 +49: [2023-04-29 09:45:16,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +49: [2023-04-29 09:45:16,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +61: [2023-04-29 09:45:16,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +61: [2023-04-29 09:45:16,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +61: [2023-04-29 09:45:16,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +61: [2023-04-29 09:45:16,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,538] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 + 5: [2023-04-29 09:45:16,541] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 + 2: [2023-04-29 09:45:16,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. + 2: [2023-04-29 09:45:16,544] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 + 2: [2023-04-29 09:45:16,545] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +50: [2023-04-29 09:45:16,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +50: [2023-04-29 09:45:16,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +50: [2023-04-29 09:45:16,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +50: [2023-04-29 09:45:16,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,551] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 +60: [2023-04-29 09:45:16,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +38: [2023-04-29 09:45:16,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:16,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +38: [2023-04-29 09:45:16,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 3: [2023-04-29 09:45:16,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,555] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 +60: [2023-04-29 09:45:16,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +12: [2023-04-29 09:45:16,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +12: [2023-04-29 09:45:16,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +12: [2023-04-29 09:45:16,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +38: [2023-04-29 09:45:16,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +12: [2023-04-29 09:45:16,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +38: [2023-04-29 09:45:16,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +31: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +38: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +39: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +31: [2023-04-29 09:45:16,558] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 +53: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:16,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +30: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +38: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +53: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +30: [2023-04-29 09:45:16,559] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 +38: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +38: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +39: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +39: [2023-04-29 09:45:16,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +14: [2023-04-29 09:45:16,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +14: [2023-04-29 09:45:16,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +14: [2023-04-29 09:45:16,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +53: [2023-04-29 09:45:16,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +14: [2023-04-29 09:45:16,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +39: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 1: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 1: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 1: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +39: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... + 1: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +53: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +39: [2023-04-29 09:45:16,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +39: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +39: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... + 4: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 4: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 4: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +39: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +39: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +39: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. + 4: [2023-04-29 09:45:16,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,562] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +39: [2023-04-29 09:45:16,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +53: [2023-04-29 09:45:16,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +53: [2023-04-29 09:45:16,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:16,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +61: [2023-04-29 09:45:16,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:16,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:16,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +61: [2023-04-29 09:45:16,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +53: [2023-04-29 09:45:16,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +31: [2023-04-29 09:45:16,565] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 +61: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +11: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +11: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +11: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +31: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +11: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +31: [2023-04-29 09:45:16,566] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 +53: [2023-04-29 09:45:16,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:16,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +53: [2023-04-29 09:45:16,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +53: [2023-04-29 09:45:16,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +49: [2023-04-29 09:45:16,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +30: [2023-04-29 09:45:16,567] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +61: [2023-04-29 09:45:16,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +10: [2023-04-29 09:45:16,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +10: [2023-04-29 09:45:16,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +10: [2023-04-29 09:45:16,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +10: [2023-04-29 09:45:16,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +27: [2023-04-29 09:45:16,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-04-29 09:45:16,572] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 +49: [2023-04-29 09:45:16,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +49: [2023-04-29 09:45:16,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +49: [2023-04-29 09:45:16,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +31: [2023-04-29 09:45:16,573] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 +31: [2023-04-29 09:45:16,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-04-29 09:45:16,577] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 +50: [2023-04-29 09:45:16,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:16,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +27: [2023-04-29 09:45:16,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +50: [2023-04-29 09:45:16,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +31: [2023-04-29 09:45:16,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-04-29 09:45:16,582] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 +13: [2023-04-29 09:45:16,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +50: [2023-04-29 09:45:16,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +13: [2023-04-29 09:45:16,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +50: [2023-04-29 09:45:16,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +31: [2023-04-29 09:45:16,584] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 +13: [2023-04-29 09:45:16,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +50: [2023-04-29 09:45:16,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:16,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +13: [2023-04-29 09:45:16,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +50: [2023-04-29 09:45:16,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +27: [2023-04-29 09:45:16,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-04-29 09:45:16,586] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 +50: [2023-04-29 09:45:16,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +50: [2023-04-29 09:45:16,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:16,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +50: [2023-04-29 09:45:16,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +50: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +36: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +50: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +50: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +36: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +36: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +50: [2023-04-29 09:45:16,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +31: [2023-04-29 09:45:16,589] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 +27: [2023-04-29 09:45:16,594] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 +37: [2023-04-29 09:45:16,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +37: [2023-04-29 09:45:16,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +38: [2023-04-29 09:45:16,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_03_optim_states.pt... +38: [2023-04-29 09:45:16,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_03_optim_states.pt... +37: [2023-04-29 09:45:16,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +38: [2023-04-29 09:45:16,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_03_optim_states.pt... +38: [2023-04-29 09:45:16,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_03_optim_states.pt... +37: [2023-04-29 09:45:16,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +18: [2023-04-29 09:45:16,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +18: [2023-04-29 09:45:16,601] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 +22: [2023-04-29 09:45:16,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +22: [2023-04-29 09:45:16,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +22: [2023-04-29 09:45:16,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +39: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_03_optim_states.pt... +39: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_03_optim_states.pt... +39: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_03_optim_states.pt... +39: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_03_optim_states.pt... +22: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +53: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_03_optim_states.pt... +53: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_03_optim_states.pt... +53: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_03_optim_states.pt... +53: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_03_optim_states.pt... +48: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +48: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +48: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +48: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +32: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +32: [2023-04-29 09:45:16,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +32: [2023-04-29 09:45:16,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +32: [2023-04-29 09:45:16,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:16,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +33: [2023-04-29 09:45:16,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +33: [2023-04-29 09:45:16,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +18: [2023-04-29 09:45:16,608] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 +33: [2023-04-29 09:45:16,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,610] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +60: [2023-04-29 09:45:16,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +24: [2023-04-29 09:45:16,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +24: [2023-04-29 09:45:16,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +24: [2023-04-29 09:45:16,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +20: [2023-04-29 09:45:16,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +20: [2023-04-29 09:45:16,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +20: [2023-04-29 09:45:16,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +24: [2023-04-29 09:45:16,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +20: [2023-04-29 09:45:16,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +23: [2023-04-29 09:45:16,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +23: [2023-04-29 09:45:16,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +23: [2023-04-29 09:45:16,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +23: [2023-04-29 09:45:16,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +54: [2023-04-29 09:45:16,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +54: [2023-04-29 09:45:16,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,618] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 +54: [2023-04-29 09:45:16,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +60: [2023-04-29 09:45:16,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +36: [2023-04-29 09:45:16,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +36: [2023-04-29 09:45:16,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +54: [2023-04-29 09:45:16,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +50: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_03_optim_states.pt... +50: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_03_optim_states.pt... +50: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_03_optim_states.pt... +50: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_03_optim_states.pt... + 1: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... + 1: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... + 1: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... +12: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +12: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +12: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +12: [2023-04-29 09:45:16,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... + 2: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 8: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 8: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 8: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 8: [2023-04-29 09:45:16,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +36: [2023-04-29 09:45:16,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +51: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +51: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +51: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +14: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +14: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +14: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +51: [2023-04-29 09:45:16,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +11: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +11: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +60: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +11: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +36: [2023-04-29 09:45:16,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +37: [2023-04-29 09:45:16,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:16,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:16,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +61: [2023-04-29 09:45:16,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +37: [2023-04-29 09:45:16,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:16,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +15: [2023-04-29 09:45:16,631] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +37: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +21: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +21: [2023-04-29 09:45:16,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +21: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +37: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +60: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +21: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +32: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +37: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +32: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:16,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-04-29 09:45:16,633] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 +35: [2023-04-29 09:45:16,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +35: [2023-04-29 09:45:16,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +61: [2023-04-29 09:45:16,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:16,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +32: [2023-04-29 09:45:16,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:16,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +32: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +33: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +32: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +32: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +48: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 4: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... + 4: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... + 4: [2023-04-29 09:45:16,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... +35: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +32: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +32: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +32: [2023-04-29 09:45:16,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +48: [2023-04-29 09:45:16,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +15: [2023-04-29 09:45:16,638] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +49: [2023-04-29 09:45:16,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:16,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +48: [2023-04-29 09:45:16,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +48: [2023-04-29 09:45:16,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +61: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +10: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +32: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +48: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +10: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... +48: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +33: [2023-04-29 09:45:16,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:16,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +32: [2023-04-29 09:45:16,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +32: [2023-04-29 09:45:16,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +33: [2023-04-29 09:45:16,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +32: [2023-04-29 09:45:16,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +32: [2023-04-29 09:45:16,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +48: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +33: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +48: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +48: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +48: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +48: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +48: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +49: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +49: [2023-04-29 09:45:16,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +48: [2023-04-29 09:45:16,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +22: [2023-04-29 09:45:16,642] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +61: [2023-04-29 09:45:16,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +33: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +33: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +56: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +33: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +33: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +33: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +56: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +49: [2023-04-29 09:45:16,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +13: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... +13: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +13: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +49: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +33: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +61: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +56: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +33: [2023-04-29 09:45:16,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +49: [2023-04-29 09:45:16,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +49: [2023-04-29 09:45:16,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... + 5: [2023-04-29 09:45:16,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +54: [2023-04-29 09:45:16,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... + 5: [2023-04-29 09:45:16,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +54: [2023-04-29 09:45:16,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:16,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:16,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:16,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +54: [2023-04-29 09:45:16,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +49: [2023-04-29 09:45:16,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +54: [2023-04-29 09:45:16,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:16,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:16,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +54: [2023-04-29 09:45:16,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +54: [2023-04-29 09:45:16,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +54: [2023-04-29 09:45:16,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +54: [2023-04-29 09:45:16,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +54: [2023-04-29 09:45:16,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +54: [2023-04-29 09:45:16,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +54: [2023-04-29 09:45:16,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +51: [2023-04-29 09:45:16,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +51: [2023-04-29 09:45:16,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:16,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +51: [2023-04-29 09:45:16,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +51: [2023-04-29 09:45:16,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:16,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... + 3: [2023-04-29 09:45:16,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,658] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 +25: [2023-04-29 09:45:16,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +25: [2023-04-29 09:45:16,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +51: [2023-04-29 09:45:16,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +25: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +25: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +51: [2023-04-29 09:45:16,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +27: [2023-04-29 09:45:16,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +27: [2023-04-29 09:45:16,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +27: [2023-04-29 09:45:16,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +51: [2023-04-29 09:45:16,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +27: [2023-04-29 09:45:16,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +51: [2023-04-29 09:45:16,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +35: [2023-04-29 09:45:16,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:16,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:16,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:16,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:16,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +35: [2023-04-29 09:45:16,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +35: [2023-04-29 09:45:16,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +35: [2023-04-29 09:45:16,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,667] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 +28: [2023-04-29 09:45:16,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +28: [2023-04-29 09:45:16,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +28: [2023-04-29 09:45:16,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +28: [2023-04-29 09:45:16,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +35: [2023-04-29 09:45:16,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:16,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:16,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +56: [2023-04-29 09:45:16,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +35: [2023-04-29 09:45:16,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +56: [2023-04-29 09:45:16,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +22: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... +22: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +56: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +56: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +22: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +37: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_03_optim_states.pt... +37: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_03_optim_states.pt... +37: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_03_optim_states.pt... +37: [2023-04-29 09:45:16,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_03_optim_states.pt... +56: [2023-04-29 09:45:16,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +56: [2023-04-29 09:45:16,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +35: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +24: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +24: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +24: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +35: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +35: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +35: [2023-04-29 09:45:16,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +56: [2023-04-29 09:45:16,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:16,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +56: [2023-04-29 09:45:16,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +56: [2023-04-29 09:45:16,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:16,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +56: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +56: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +56: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +56: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +56: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +23: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +23: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +23: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +23: [2023-04-29 09:45:16,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +36: [2023-04-29 09:45:16,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +20: [2023-04-29 09:45:16,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +20: [2023-04-29 09:45:16,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +20: [2023-04-29 09:45:16,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +20: [2023-04-29 09:45:16,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +32: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... +32: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... +32: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... +32: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... +48: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_03_optim_states.pt... +48: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_03_optim_states.pt... +48: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_03_optim_states.pt... +48: [2023-04-29 09:45:16,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_03_optim_states.pt... +33: [2023-04-29 09:45:16,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_03_optim_states.pt... +33: [2023-04-29 09:45:16,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_03_optim_states.pt... +33: [2023-04-29 09:45:16,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_03_optim_states.pt... +33: [2023-04-29 09:45:16,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_03_optim_states.pt... + 6: [2023-04-29 09:45:16,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 6: [2023-04-29 09:45:16,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 6: [2023-04-29 09:45:16,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 6: [2023-04-29 09:45:16,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +36: [2023-04-29 09:45:16,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +62: [2023-04-29 09:45:16,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +62: [2023-04-29 09:45:16,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +62: [2023-04-29 09:45:16,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +62: [2023-04-29 09:45:16,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +36: [2023-04-29 09:45:16,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 7: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 7: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 7: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 7: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 8: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... + 8: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... + 8: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... + 8: [2023-04-29 09:45:16,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... +40: [2023-04-29 09:45:16,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +40: [2023-04-29 09:45:16,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +40: [2023-04-29 09:45:16,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +40: [2023-04-29 09:45:16,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 2: [2023-04-29 09:45:16,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-04-29 09:45:16,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... +36: [2023-04-29 09:45:16,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +36: [2023-04-29 09:45:16,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. + 2: [2023-04-29 09:45:16,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... + 2: [2023-04-29 09:45:16,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... +54: [2023-04-29 09:45:16,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_03_optim_states.pt... +54: [2023-04-29 09:45:16,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_03_optim_states.pt... +54: [2023-04-29 09:45:16,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_03_optim_states.pt... +54: [2023-04-29 09:45:16,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_03_optim_states.pt... +36: [2023-04-29 09:45:16,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +15: [2023-04-29 09:45:16,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +15: [2023-04-29 09:45:16,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +15: [2023-04-29 09:45:16,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +15: [2023-04-29 09:45:16,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +21: [2023-04-29 09:45:16,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +21: [2023-04-29 09:45:16,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-04-29 09:45:16,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +21: [2023-04-29 09:45:16,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +36: [2023-04-29 09:45:16,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +51: [2023-04-29 09:45:16,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_03_optim_states.pt... +51: [2023-04-29 09:45:16,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_03_optim_states.pt... +51: [2023-04-29 09:45:16,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_03_optim_states.pt... +51: [2023-04-29 09:45:16,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_03_optim_states.pt... + 9: [2023-04-29 09:45:16,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 9: [2023-04-29 09:45:16,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +36: [2023-04-29 09:45:16,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +35: [2023-04-29 09:45:16,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_03_optim_states.pt... +35: [2023-04-29 09:45:16,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_03_optim_states.pt... +35: [2023-04-29 09:45:16,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_03_optim_states.pt... +35: [2023-04-29 09:45:16,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_03_optim_states.pt... +62: [2023-04-29 09:45:16,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:16,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:16,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +30: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +30: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +56: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_03_optim_states.pt... +56: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_03_optim_states.pt... +56: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_03_optim_states.pt... +56: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_03_optim_states.pt... +30: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +62: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +30: [2023-04-29 09:45:16,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +43: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +43: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +43: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +43: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 5: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 5: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... + 5: [2023-04-29 09:45:16,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... +62: [2023-04-29 09:45:16,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:16,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:16,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:16,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +62: [2023-04-29 09:45:16,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:16,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +36: [2023-04-29 09:45:16,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_02_optim_states.pt. +40: [2023-04-29 09:45:16,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:16,720] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 290 +40: [2023-04-29 09:45:16,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:16,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +62: [2023-04-29 09:45:16,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +40: [2023-04-29 09:45:16,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:16,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:16,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +17: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +17: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +62: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +62: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +17: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +17: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +40: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +40: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +62: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +26: [2023-04-29 09:45:16,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +26: [2023-04-29 09:45:16,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +26: [2023-04-29 09:45:16,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +40: [2023-04-29 09:45:16,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +26: [2023-04-29 09:45:16,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +40: [2023-04-29 09:45:16,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:16,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +40: [2023-04-29 09:45:16,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +40: [2023-04-29 09:45:16,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +40: [2023-04-29 09:45:16,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +40: [2023-04-29 09:45:16,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +40: [2023-04-29 09:45:16,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +40: [2023-04-29 09:45:16,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +40: [2023-04-29 09:45:16,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +25: [2023-04-29 09:45:16,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... +25: [2023-04-29 09:45:16,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-04-29 09:45:16,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +25: [2023-04-29 09:45:16,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... +27: [2023-04-29 09:45:16,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +27: [2023-04-29 09:45:16,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +27: [2023-04-29 09:45:16,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-04-29 09:45:16,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +36: [2023-04-29 09:45:16,727] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 290 + 3: [2023-04-29 09:45:16,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 3: [2023-04-29 09:45:16,736] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +52: [2023-04-29 09:45:16,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +52: [2023-04-29 09:45:16,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +52: [2023-04-29 09:45:16,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +52: [2023-04-29 09:45:16,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_43-model_01-model_states.pt. +29: [2023-04-29 09:45:16,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +29: [2023-04-29 09:45:16,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +29: [2023-04-29 09:45:16,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +29: [2023-04-29 09:45:16,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +28: [2023-04-29 09:45:16,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +28: [2023-04-29 09:45:16,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-04-29 09:45:16,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +28: [2023-04-29 09:45:16,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... +43: [2023-04-29 09:45:16,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +19: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +19: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +19: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +19: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +43: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 6: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 6: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-04-29 09:45:16,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... + 3: [2023-04-29 09:45:16,746] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 +43: [2023-04-29 09:45:16,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:16,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +43: [2023-04-29 09:45:16,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:16,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +43: [2023-04-29 09:45:16,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +62: [2023-04-29 09:45:16,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_03_optim_states.pt... +62: [2023-04-29 09:45:16,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_03_optim_states.pt... +62: [2023-04-29 09:45:16,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_03_optim_states.pt... +62: [2023-04-29 09:45:16,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_03_optim_states.pt... +43: [2023-04-29 09:45:16,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:16,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +43: [2023-04-29 09:45:16,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +43: [2023-04-29 09:45:16,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +43: [2023-04-29 09:45:16,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +43: [2023-04-29 09:45:16,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +43: [2023-04-29 09:45:16,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +43: [2023-04-29 09:45:16,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +63: [2023-04-29 09:45:16,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +43: [2023-04-29 09:45:16,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +63: [2023-04-29 09:45:16,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +63: [2023-04-29 09:45:16,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +63: [2023-04-29 09:45:16,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +31: [2023-04-29 09:45:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +31: [2023-04-29 09:45:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +31: [2023-04-29 09:45:16,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +31: [2023-04-29 09:45:16,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +18: [2023-04-29 09:45:16,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +18: [2023-04-29 09:45:16,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +18: [2023-04-29 09:45:16,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +18: [2023-04-29 09:45:16,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +40: [2023-04-29 09:45:16,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_03_optim_states.pt... +40: [2023-04-29 09:45:16,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_03_optim_states.pt... +40: [2023-04-29 09:45:16,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_03_optim_states.pt... +40: [2023-04-29 09:45:16,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_03_optim_states.pt... +15: [2023-04-29 09:45:16,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +15: [2023-04-29 09:45:16,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +15: [2023-04-29 09:45:16,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-04-29 09:45:16,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +52: [2023-04-29 09:45:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:16,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... +52: [2023-04-29 09:45:16,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt... + 9: [2023-04-29 09:45:16,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... + 9: [2023-04-29 09:45:16,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 9: [2023-04-29 09:45:16,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-04-29 09:45:16,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 7: [2023-04-29 09:45:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-04-29 09:45:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... + 7: [2023-04-29 09:45:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... + 7: [2023-04-29 09:45:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... +63: [2023-04-29 09:45:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +30: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +30: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +63: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +16: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +16: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +16: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +30: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... +16: [2023-04-29 09:45:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +63: [2023-04-29 09:45:16,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +63: [2023-04-29 09:45:16,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:16,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +63: [2023-04-29 09:45:16,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +63: [2023-04-29 09:45:16,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:16,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +63: [2023-04-29 09:45:16,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:16,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +44: [2023-04-29 09:45:16,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +63: [2023-04-29 09:45:16,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +63: [2023-04-29 09:45:16,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +63: [2023-04-29 09:45:16,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +63: [2023-04-29 09:45:16,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +63: [2023-04-29 09:45:16,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +63: [2023-04-29 09:45:16,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +44: [2023-04-29 09:45:16,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +44: [2023-04-29 09:45:16,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +43: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_03_optim_states.pt... +43: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_03_optim_states.pt... +43: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_03_optim_states.pt... +43: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_03_optim_states.pt... +55: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +55: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +55: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +55: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +47: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +47: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +47: [2023-04-29 09:45:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +47: [2023-04-29 09:45:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +45: [2023-04-29 09:45:16,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +45: [2023-04-29 09:45:16,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +45: [2023-04-29 09:45:16,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +45: [2023-04-29 09:45:16,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +26: [2023-04-29 09:45:16,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +26: [2023-04-29 09:45:16,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +26: [2023-04-29 09:45:16,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-04-29 09:45:16,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +17: [2023-04-29 09:45:16,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-04-29 09:45:16,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-04-29 09:45:16,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... +17: [2023-04-29 09:45:16,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +29: [2023-04-29 09:45:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-04-29 09:45:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +29: [2023-04-29 09:45:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-04-29 09:45:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +44: [2023-04-29 09:45:16,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:16,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:16,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:16,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:16,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +19: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +19: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... +19: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +19: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +44: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +44: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +44: [2023-04-29 09:45:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +44: [2023-04-29 09:45:16,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +44: [2023-04-29 09:45:16,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +44: [2023-04-29 09:45:16,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +44: [2023-04-29 09:45:16,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +44: [2023-04-29 09:45:16,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +44: [2023-04-29 09:45:16,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +44: [2023-04-29 09:45:16,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +45: [2023-04-29 09:45:16,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +45: [2023-04-29 09:45:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +34: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +55: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +47: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +55: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +47: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +47: [2023-04-29 09:45:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +45: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +45: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +45: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +55: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +55: [2023-04-29 09:45:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +55: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +45: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +45: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +45: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +47: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +47: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +47: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +45: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +47: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +55: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +55: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +55: [2023-04-29 09:45:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +55: [2023-04-29 09:45:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +55: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +55: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +55: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +55: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +46: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +46: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +46: [2023-04-29 09:45:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +46: [2023-04-29 09:45:16,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +59: [2023-04-29 09:45:16,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +59: [2023-04-29 09:45:16,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +63: [2023-04-29 09:45:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_03_optim_states.pt... +63: [2023-04-29 09:45:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_03_optim_states.pt... +63: [2023-04-29 09:45:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_03_optim_states.pt... +63: [2023-04-29 09:45:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_03_optim_states.pt... +59: [2023-04-29 09:45:16,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +59: [2023-04-29 09:45:16,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 3: [2023-04-29 09:45:16,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... + 3: [2023-04-29 09:45:16,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... + 3: [2023-04-29 09:45:16,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-04-29 09:45:16,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... +31: [2023-04-29 09:45:16,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-04-29 09:45:16,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-04-29 09:45:16,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +31: [2023-04-29 09:45:16,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... +52: [2023-04-29 09:45:16,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +18: [2023-04-29 09:45:16,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-04-29 09:45:16,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +18: [2023-04-29 09:45:16,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +18: [2023-04-29 09:45:16,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +52: [2023-04-29 09:45:16,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +34: [2023-04-29 09:45:16,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:16,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:16,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +34: [2023-04-29 09:45:16,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +52: [2023-04-29 09:45:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +52: [2023-04-29 09:45:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_00-model_states.pt. +34: [2023-04-29 09:45:16,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +34: [2023-04-29 09:45:16,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +34: [2023-04-29 09:45:16,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +34: [2023-04-29 09:45:16,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +34: [2023-04-29 09:45:16,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +34: [2023-04-29 09:45:16,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +34: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +46: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +16: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... +34: [2023-04-29 09:45:16,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +59: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +52: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt... +46: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:16,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:16,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +46: [2023-04-29 09:45:16,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:16,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +46: [2023-04-29 09:45:16,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:16,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +59: [2023-04-29 09:45:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +46: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +46: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +46: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +59: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +46: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +46: [2023-04-29 09:45:16,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +59: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +44: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_03_optim_states.pt... +44: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_03_optim_states.pt... +44: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_03_optim_states.pt... +44: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_03_optim_states.pt... +59: [2023-04-29 09:45:16,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +46: [2023-04-29 09:45:16,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +59: [2023-04-29 09:45:16,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +59: [2023-04-29 09:45:16,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +59: [2023-04-29 09:45:16,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +59: [2023-04-29 09:45:16,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +47: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_03_optim_states.pt... +45: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_03_optim_states.pt... +45: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_03_optim_states.pt... +45: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_03_optim_states.pt... +45: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_03_optim_states.pt... +47: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_03_optim_states.pt... +47: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_03_optim_states.pt... +47: [2023-04-29 09:45:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_03_optim_states.pt... +55: [2023-04-29 09:45:16,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_03_optim_states.pt... +55: [2023-04-29 09:45:16,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_03_optim_states.pt... +55: [2023-04-29 09:45:16,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_03_optim_states.pt... +55: [2023-04-29 09:45:16,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_03_optim_states.pt... +60: [2023-04-29 09:45:16,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. + 0: [2023-04-29 09:45:16,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_23-model_01-model_states.pt. +57: [2023-04-29 09:45:16,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +57: [2023-04-29 09:45:16,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +57: [2023-04-29 09:45:16,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +57: [2023-04-29 09:45:16,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +34: [2023-04-29 09:45:16,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_03_optim_states.pt... +34: [2023-04-29 09:45:16,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_03_optim_states.pt... +34: [2023-04-29 09:45:16,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_03_optim_states.pt... +34: [2023-04-29 09:45:16,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_03_optim_states.pt... +61: [2023-04-29 09:45:16,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +61: [2023-04-29 09:45:16,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +61: [2023-04-29 09:45:16,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +61: [2023-04-29 09:45:16,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +46: [2023-04-29 09:45:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_03_optim_states.pt... +46: [2023-04-29 09:45:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_03_optim_states.pt... +46: [2023-04-29 09:45:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_03_optim_states.pt... +46: [2023-04-29 09:45:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_03_optim_states.pt... +59: [2023-04-29 09:45:16,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_03_optim_states.pt... +59: [2023-04-29 09:45:16,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_03_optim_states.pt... +59: [2023-04-29 09:45:16,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_03_optim_states.pt... +59: [2023-04-29 09:45:16,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_03_optim_states.pt... +60: [2023-04-29 09:45:16,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:16,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:16,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:16,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:16,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +60: [2023-04-29 09:45:16,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +49: [2023-04-29 09:45:16,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +49: [2023-04-29 09:45:16,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +49: [2023-04-29 09:45:16,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +49: [2023-04-29 09:45:16,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +60: [2023-04-29 09:45:16,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +60: [2023-04-29 09:45:16,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +60: [2023-04-29 09:45:16,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:16,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +60: [2023-04-29 09:45:16,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:16,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +60: [2023-04-29 09:45:16,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:16,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +60: [2023-04-29 09:45:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:16,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +57: [2023-04-29 09:45:16,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +60: [2023-04-29 09:45:16,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +57: [2023-04-29 09:45:16,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +57: [2023-04-29 09:45:16,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +57: [2023-04-29 09:45:16,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:16,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:16,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +57: [2023-04-29 09:45:16,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +57: [2023-04-29 09:45:16,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +57: [2023-04-29 09:45:16,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +57: [2023-04-29 09:45:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +57: [2023-04-29 09:45:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +57: [2023-04-29 09:45:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +57: [2023-04-29 09:45:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +61: [2023-04-29 09:45:16,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:16,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:16,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +41: [2023-04-29 09:45:16,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_03_optim_states.pt. +41: [2023-04-29 09:45:16,919] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 335 +61: [2023-04-29 09:45:16,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +11: [2023-04-29 09:45:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +61: [2023-04-29 09:45:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:16,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +11: [2023-04-29 09:45:16,920] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 +18: [2023-04-29 09:45:16,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +18: [2023-04-29 09:45:16,922] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 +61: [2023-04-29 09:45:16,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:16,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +61: [2023-04-29 09:45:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +61: [2023-04-29 09:45:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +61: [2023-04-29 09:45:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:16,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +61: [2023-04-29 09:45:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. + 8: [2023-04-29 09:45:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. + 8: [2023-04-29 09:45:16,924] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 +61: [2023-04-29 09:45:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +61: [2023-04-29 09:45:16,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +61: [2023-04-29 09:45:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +41: [2023-04-29 09:45:16,926] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 335 +18: [2023-04-29 09:45:16,928] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +11: [2023-04-29 09:45:16,928] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 +49: [2023-04-29 09:45:16,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +49: [2023-04-29 09:45:16,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +49: [2023-04-29 09:45:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +49: [2023-04-29 09:45:16,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. + 8: [2023-04-29 09:45:16,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 +49: [2023-04-29 09:45:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +49: [2023-04-29 09:45:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +49: [2023-04-29 09:45:16,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +49: [2023-04-29 09:45:16,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +49: [2023-04-29 09:45:16,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +49: [2023-04-29 09:45:16,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +49: [2023-04-29 09:45:16,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +49: [2023-04-29 09:45:16,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +49: [2023-04-29 09:45:16,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +49: [2023-04-29 09:45:16,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +60: [2023-04-29 09:45:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_03_optim_states.pt... +60: [2023-04-29 09:45:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_03_optim_states.pt... +60: [2023-04-29 09:45:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_03_optim_states.pt... +60: [2023-04-29 09:45:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_03_optim_states.pt... +41: [2023-04-29 09:45:16,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_03_optim_states.pt. +58: [2023-04-29 09:45:16,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_03_optim_states.pt. +58: [2023-04-29 09:45:16,952] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 471 +41: [2023-04-29 09:45:16,952] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 333 +42: [2023-04-29 09:45:16,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_03_optim_states.pt. +42: [2023-04-29 09:45:16,953] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 341 +42: [2023-04-29 09:45:16,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_03_optim_states.pt. +42: [2023-04-29 09:45:16,955] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 339 +57: [2023-04-29 09:45:16,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_03_optim_states.pt... +57: [2023-04-29 09:45:16,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_03_optim_states.pt... +57: [2023-04-29 09:45:16,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_03_optim_states.pt... +57: [2023-04-29 09:45:16,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_03_optim_states.pt... + 0: [2023-04-29 09:45:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... + 0: [2023-04-29 09:45:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... + 0: [2023-04-29 09:45:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-04-29 09:45:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... +58: [2023-04-29 09:45:16,959] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 471 +41: [2023-04-29 09:45:16,960] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 333 +42: [2023-04-29 09:45:16,962] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 341 +42: [2023-04-29 09:45:16,963] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 339 +19: [2023-04-29 09:45:16,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-04-29 09:45:16,964] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 +60: [2023-04-29 09:45:16,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_02_optim_states.pt. +60: [2023-04-29 09:45:16,969] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 484 +61: [2023-04-29 09:45:16,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_03_optim_states.pt... +61: [2023-04-29 09:45:16,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_03_optim_states.pt... +61: [2023-04-29 09:45:16,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_03_optim_states.pt... +61: [2023-04-29 09:45:16,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_03_optim_states.pt... +19: [2023-04-29 09:45:16,974] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 +60: [2023-04-29 09:45:16,976] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 484 +49: [2023-04-29 09:45:16,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_03_optim_states.pt... +49: [2023-04-29 09:45:16,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_03_optim_states.pt... +49: [2023-04-29 09:45:16,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_03_optim_states.pt... +49: [2023-04-29 09:45:16,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_03_optim_states.pt... +58: [2023-04-29 09:45:16,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_03_optim_states.pt. +58: [2023-04-29 09:45:16,997] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 465 +12: [2023-04-29 09:45:17,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +53: [2023-04-29 09:45:17,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_03_optim_states.pt. +54: [2023-04-29 09:45:17,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_02_optim_states.pt. +52: [2023-04-29 09:45:17,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +52: [2023-04-29 09:45:17,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +10: [2023-04-29 09:45:17,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +58: [2023-04-29 09:45:17,004] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 465 +13: [2023-04-29 09:45:17,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +39: [2023-04-29 09:45:17,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_03_optim_states.pt. +12: [2023-04-29 09:45:17,000] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 +14: [2023-04-29 09:45:17,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +36: [2023-04-29 09:45:17,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +36: [2023-04-29 09:45:17,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 4: [2023-04-29 09:45:17,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. +42: [2023-04-29 09:45:17,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_03_optim_states.pt. + 1: [2023-04-29 09:45:17,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. +53: [2023-04-29 09:45:17,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 431 +54: [2023-04-29 09:45:17,023] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 432 +52: [2023-04-29 09:45:17,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +10: [2023-04-29 09:45:17,056] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 +13: [2023-04-29 09:45:17,056] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 +39: [2023-04-29 09:45:17,050] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 317 +12: [2023-04-29 09:45:17,007] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 +14: [2023-04-29 09:45:17,025] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 +36: [2023-04-29 09:45:17,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. + 4: [2023-04-29 09:45:17,020] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 +42: [2023-04-29 09:45:17,018] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 337 + 1: [2023-04-29 09:45:17,043] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 +53: [2023-04-29 09:45:17,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_03_optim_states.pt. +54: [2023-04-29 09:45:17,030] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 432 +52: [2023-04-29 09:45:17,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +10: [2023-04-29 09:45:17,064] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 +13: [2023-04-29 09:45:17,063] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 +39: [2023-04-29 09:45:17,057] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 317 +12: [2023-04-29 09:45:17,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +14: [2023-04-29 09:45:17,033] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +36: [2023-04-29 09:45:17,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_44-model_01-model_states.pt. +42: [2023-04-29 09:45:17,025] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 337 + 1: [2023-04-29 09:45:17,051] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +53: [2023-04-29 09:45:17,070] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 425 +10: [2023-04-29 09:45:17,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +13: [2023-04-29 09:45:17,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +12: [2023-04-29 09:45:17,072] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 +36: [2023-04-29 09:45:17,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +53: [2023-04-29 09:45:17,072] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 431 +10: [2023-04-29 09:45:17,067] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 +13: [2023-04-29 09:45:17,063] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +36: [2023-04-29 09:45:17,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +10: [2023-04-29 09:45:17,075] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 +13: [2023-04-29 09:45:17,071] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +36: [2023-04-29 09:45:17,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:17,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:17,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +36: [2023-04-29 09:45:17,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +36: [2023-04-29 09:45:17,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:17,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:17,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +36: [2023-04-29 09:45:17,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +36: [2023-04-29 09:45:17,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +36: [2023-04-29 09:45:17,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +36: [2023-04-29 09:45:17,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +36: [2023-04-29 09:45:17,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +36: [2023-04-29 09:45:17,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +36: [2023-04-29 09:45:17,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +53: [2023-04-29 09:45:17,077] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 425 +52: [2023-04-29 09:45:17,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:17,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:17,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +12: [2023-04-29 09:45:17,079] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 +22: [2023-04-29 09:45:17,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-04-29 09:45:17,080] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +29: [2023-04-29 09:45:17,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +29: [2023-04-29 09:45:17,082] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt... +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_00-model_states.pt. +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +52: [2023-04-29 09:45:17,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt... +52: [2023-04-29 09:45:17,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +52: [2023-04-29 09:45:17,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. +52: [2023-04-29 09:45:17,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/layer_46-model_01-model_states.pt. + 4: [2023-04-29 09:45:17,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-04-29 09:45:17,088] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 +22: [2023-04-29 09:45:17,088] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 + 4: [2023-04-29 09:45:17,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 +29: [2023-04-29 09:45:17,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 +14: [2023-04-29 09:45:17,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-04-29 09:45:17,091] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 +12: [2023-04-29 09:45:17,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-04-29 09:45:17,092] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +48: [2023-04-29 09:45:17,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_03_optim_states.pt. +48: [2023-04-29 09:45:17,094] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 391 + 4: [2023-04-29 09:45:17,095] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 +14: [2023-04-29 09:45:17,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +38: [2023-04-29 09:45:17,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_03_optim_states.pt. +14: [2023-04-29 09:45:17,098] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +38: [2023-04-29 09:45:17,098] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 311 +38: [2023-04-29 09:45:17,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_03_optim_states.pt. +38: [2023-04-29 09:45:17,098] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 307 +41: [2023-04-29 09:45:17,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_03_optim_states.pt. +41: [2023-04-29 09:45:17,099] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 331 +12: [2023-04-29 09:45:17,099] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 +14: [2023-04-29 09:45:17,099] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 +53: [2023-04-29 09:45:17,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_03_optim_states.pt. +53: [2023-04-29 09:45:17,100] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 427 +10: [2023-04-29 09:45:17,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +23: [2023-04-29 09:45:17,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-04-29 09:45:17,101] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 +10: [2023-04-29 09:45:17,101] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +48: [2023-04-29 09:45:17,101] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 391 +13: [2023-04-29 09:45:17,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +14: [2023-04-29 09:45:17,104] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 +13: [2023-04-29 09:45:17,104] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +41: [2023-04-29 09:45:17,106] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 331 +53: [2023-04-29 09:45:17,107] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 427 +10: [2023-04-29 09:45:17,108] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 +38: [2023-04-29 09:45:17,108] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 307 +38: [2023-04-29 09:45:17,108] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 311 +23: [2023-04-29 09:45:17,108] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 +36: [2023-04-29 09:45:17,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_03_optim_states.pt... +36: [2023-04-29 09:45:17,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_03_optim_states.pt... +36: [2023-04-29 09:45:17,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_03_optim_states.pt... +36: [2023-04-29 09:45:17,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_03_optim_states.pt... +13: [2023-04-29 09:45:17,111] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +32: [2023-04-29 09:45:17,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. +32: [2023-04-29 09:45:17,120] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 259 +53: [2023-04-29 09:45:17,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_03_optim_states.pt. +53: [2023-04-29 09:45:17,121] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 429 + 9: [2023-04-29 09:45:17,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. + 9: [2023-04-29 09:45:17,123] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 +23: [2023-04-29 09:45:17,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-04-29 09:45:17,124] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +20: [2023-04-29 09:45:17,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +20: [2023-04-29 09:45:17,125] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 +52: [2023-04-29 09:45:17,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_03_optim_states.pt... +52: [2023-04-29 09:45:17,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_03_optim_states.pt... +52: [2023-04-29 09:45:17,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_03_optim_states.pt... +52: [2023-04-29 09:45:17,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_03_optim_states.pt... +53: [2023-04-29 09:45:17,128] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 429 +32: [2023-04-29 09:45:17,128] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 259 +24: [2023-04-29 09:45:17,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +13: [2023-04-29 09:45:17,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +24: [2023-04-29 09:45:17,128] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +13: [2023-04-29 09:45:17,128] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 + 9: [2023-04-29 09:45:17,130] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +22: [2023-04-29 09:45:17,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-04-29 09:45:17,131] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +20: [2023-04-29 09:45:17,133] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 +23: [2023-04-29 09:45:17,134] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 +24: [2023-04-29 09:45:17,136] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 +13: [2023-04-29 09:45:17,136] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 +14: [2023-04-29 09:45:17,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-04-29 09:45:17,139] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +22: [2023-04-29 09:45:17,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 + 2: [2023-04-29 09:45:17,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-04-29 09:45:17,140] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 +54: [2023-04-29 09:45:17,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_88_mp_rank_03_optim_states.pt. + 1: [2023-04-29 09:45:17,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. +54: [2023-04-29 09:45:17,140] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 433 + 1: [2023-04-29 09:45:17,141] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +38: [2023-04-29 09:45:17,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_03_optim_states.pt. +38: [2023-04-29 09:45:17,141] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 309 +32: [2023-04-29 09:45:17,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. +32: [2023-04-29 09:45:17,143] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 261 +50: [2023-04-29 09:45:17,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_03_optim_states.pt. +11: [2023-04-29 09:45:17,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +50: [2023-04-29 09:45:17,144] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 401 +11: [2023-04-29 09:45:17,144] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 +54: [2023-04-29 09:45:17,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 433 + 2: [2023-04-29 09:45:17,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 +14: [2023-04-29 09:45:17,148] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 + 1: [2023-04-29 09:45:17,148] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +39: [2023-04-29 09:45:17,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_03_optim_states.pt. +20: [2023-04-29 09:45:17,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +39: [2023-04-29 09:45:17,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 315 +20: [2023-04-29 09:45:17,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +38: [2023-04-29 09:45:17,149] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 309 +11: [2023-04-29 09:45:17,150] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 +32: [2023-04-29 09:45:17,152] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 261 +50: [2023-04-29 09:45:17,152] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 401 + 4: [2023-04-29 09:45:17,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 4: [2023-04-29 09:45:17,153] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 +50: [2023-04-29 09:45:17,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_03_optim_states.pt. +50: [2023-04-29 09:45:17,154] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 403 +39: [2023-04-29 09:45:17,155] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 315 +11: [2023-04-29 09:45:17,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +20: [2023-04-29 09:45:17,157] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +11: [2023-04-29 09:45:17,157] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 +11: [2023-04-29 09:45:17,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +11: [2023-04-29 09:45:17,158] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 + 4: [2023-04-29 09:45:17,161] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 +50: [2023-04-29 09:45:17,162] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 403 + 5: [2023-04-29 09:45:17,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +39: [2023-04-29 09:45:17,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_03_optim_states.pt. + 5: [2023-04-29 09:45:17,163] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 +39: [2023-04-29 09:45:17,163] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 319 +11: [2023-04-29 09:45:17,164] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 +11: [2023-04-29 09:45:17,165] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 + 5: [2023-04-29 09:45:17,169] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +33: [2023-04-29 09:45:17,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_4_mp_rank_03_optim_states.pt. +33: [2023-04-29 09:45:17,170] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 265 +22: [2023-04-29 09:45:17,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +39: [2023-04-29 09:45:17,171] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 319 +22: [2023-04-29 09:45:17,171] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 +32: [2023-04-29 09:45:17,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. + 8: [2023-04-29 09:45:17,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-04-29 09:45:17,173] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 +32: [2023-04-29 09:45:17,173] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 257 +58: [2023-04-29 09:45:17,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_03_optim_states.pt. +58: [2023-04-29 09:45:17,174] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 467 +37: [2023-04-29 09:45:17,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_03_optim_states.pt. +37: [2023-04-29 09:45:17,174] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 301 +17: [2023-04-29 09:45:17,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-04-29 09:45:17,175] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 +33: [2023-04-29 09:45:17,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 265 +22: [2023-04-29 09:45:17,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 +58: [2023-04-29 09:45:17,180] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 467 + 8: [2023-04-29 09:45:17,181] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 +37: [2023-04-29 09:45:17,182] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 301 +32: [2023-04-29 09:45:17,182] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 257 +20: [2023-04-29 09:45:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-04-29 09:45:17,182] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +17: [2023-04-29 09:45:17,182] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +33: [2023-04-29 09:45:17,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_03_optim_states.pt. +33: [2023-04-29 09:45:17,188] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 267 +39: [2023-04-29 09:45:17,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_03_optim_states.pt. +20: [2023-04-29 09:45:17,188] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 +11: [2023-04-29 09:45:17,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +39: [2023-04-29 09:45:17,189] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 313 +11: [2023-04-29 09:45:17,189] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 +48: [2023-04-29 09:45:17,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_03_optim_states.pt. + 5: [2023-04-29 09:45:17,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. +48: [2023-04-29 09:45:17,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 389 + 6: [2023-04-29 09:45:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. + 5: [2023-04-29 09:45:17,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 + 6: [2023-04-29 09:45:17,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 +10: [2023-04-29 09:45:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +10: [2023-04-29 09:45:17,192] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 + 9: [2023-04-29 09:45:17,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-04-29 09:45:17,194] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 +48: [2023-04-29 09:45:17,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_03_optim_states.pt. +48: [2023-04-29 09:45:17,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 387 +33: [2023-04-29 09:45:17,195] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 267 +39: [2023-04-29 09:45:17,196] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 313 +35: [2023-04-29 09:45:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_03_optim_states.pt. +28: [2023-04-29 09:45:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +35: [2023-04-29 09:45:17,196] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 287 +11: [2023-04-29 09:45:17,196] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 +28: [2023-04-29 09:45:17,196] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 + 5: [2023-04-29 09:45:17,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +10: [2023-04-29 09:45:17,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 + 8: [2023-04-29 09:45:17,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. +17: [2023-04-29 09:45:17,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. + 6: [2023-04-29 09:45:17,200] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 + 8: [2023-04-29 09:45:17,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 +17: [2023-04-29 09:45:17,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 +35: [2023-04-29 09:45:17,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_03_optim_states.pt. +35: [2023-04-29 09:45:17,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 283 + 9: [2023-04-29 09:45:17,201] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 +24: [2023-04-29 09:45:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-04-29 09:45:17,203] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +48: [2023-04-29 09:45:17,203] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 389 +51: [2023-04-29 09:45:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_03_optim_states.pt. +51: [2023-04-29 09:45:17,203] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 409 +48: [2023-04-29 09:45:17,203] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 387 +35: [2023-04-29 09:45:17,204] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 287 +28: [2023-04-29 09:45:17,204] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +17: [2023-04-29 09:45:17,207] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 + 8: [2023-04-29 09:45:17,207] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 +35: [2023-04-29 09:45:17,207] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 283 +24: [2023-04-29 09:45:17,209] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 +42: [2023-04-29 09:45:17,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_43_mp_rank_03_optim_states.pt. +42: [2023-04-29 09:45:17,210] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 343 +63: [2023-04-29 09:45:17,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_02_optim_states.pt. +63: [2023-04-29 09:45:17,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 508 +27: [2023-04-29 09:45:17,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +51: [2023-04-29 09:45:17,211] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 409 +27: [2023-04-29 09:45:17,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +33: [2023-04-29 09:45:17,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_03_optim_states.pt. +56: [2023-04-29 09:45:17,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_03_optim_states.pt. +33: [2023-04-29 09:45:17,212] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 269 +56: [2023-04-29 09:45:17,212] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 451 +21: [2023-04-29 09:45:17,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-04-29 09:45:17,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 +51: [2023-04-29 09:45:17,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_03_optim_states.pt. +51: [2023-04-29 09:45:17,215] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 415 + 8: [2023-04-29 09:45:17,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. +15: [2023-04-29 09:45:17,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. + 8: [2023-04-29 09:45:17,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 +15: [2023-04-29 09:45:17,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 +42: [2023-04-29 09:45:17,216] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 343 +63: [2023-04-29 09:45:17,218] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 508 +27: [2023-04-29 09:45:17,218] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 +56: [2023-04-29 09:45:17,219] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 451 + 5: [2023-04-29 09:45:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. +21: [2023-04-29 09:45:17,219] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 + 5: [2023-04-29 09:45:17,219] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 +24: [2023-04-29 09:45:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-04-29 09:45:17,220] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 +21: [2023-04-29 09:45:17,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +33: [2023-04-29 09:45:17,222] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 269 +21: [2023-04-29 09:45:17,222] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +51: [2023-04-29 09:45:17,222] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 415 +15: [2023-04-29 09:45:17,223] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 + 8: [2023-04-29 09:45:17,224] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 +51: [2023-04-29 09:45:17,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_03_optim_states.pt. +51: [2023-04-29 09:45:17,225] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 411 +24: [2023-04-29 09:45:17,226] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 + 5: [2023-04-29 09:45:17,226] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 +29: [2023-04-29 09:45:17,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-04-29 09:45:17,227] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 + 1: [2023-04-29 09:45:17,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-04-29 09:45:17,228] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 + 6: [2023-04-29 09:45:17,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-04-29 09:45:17,229] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 +21: [2023-04-29 09:45:17,229] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 +21: [2023-04-29 09:45:17,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-04-29 09:45:17,230] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 + 8: [2023-04-29 09:45:17,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. + 8: [2023-04-29 09:45:17,231] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 +51: [2023-04-29 09:45:17,232] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 411 + 6: [2023-04-29 09:45:17,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-04-29 09:45:17,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 +30: [2023-04-29 09:45:17,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-04-29 09:45:17,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +62: [2023-04-29 09:45:17,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_03_optim_states.pt. +62: [2023-04-29 09:45:17,234] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 499 +29: [2023-04-29 09:45:17,234] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +21: [2023-04-29 09:45:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +28: [2023-04-29 09:45:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-04-29 09:45:17,235] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +21: [2023-04-29 09:45:17,235] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 + 1: [2023-04-29 09:45:17,235] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 +21: [2023-04-29 09:45:17,237] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 + 6: [2023-04-29 09:45:17,238] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 +62: [2023-04-29 09:45:17,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_120_mp_rank_03_optim_states.pt. +62: [2023-04-29 09:45:17,238] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 497 + 8: [2023-04-29 09:45:17,238] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 + 6: [2023-04-29 09:45:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. +30: [2023-04-29 09:45:17,240] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 + 6: [2023-04-29 09:45:17,240] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 +20: [2023-04-29 09:45:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-04-29 09:45:17,240] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +62: [2023-04-29 09:45:17,242] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 499 +21: [2023-04-29 09:45:17,242] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 + 6: [2023-04-29 09:45:17,242] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 +19: [2023-04-29 09:45:17,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. +19: [2023-04-29 09:45:17,243] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 +28: [2023-04-29 09:45:17,244] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 +62: [2023-04-29 09:45:17,245] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 497 + 6: [2023-04-29 09:45:17,246] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +32: [2023-04-29 09:45:17,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. +32: [2023-04-29 09:45:17,248] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 263 +54: [2023-04-29 09:45:17,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_03_optim_states.pt. +54: [2023-04-29 09:45:17,249] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 437 +20: [2023-04-29 09:45:17,249] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 +43: [2023-04-29 09:45:17,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_47_mp_rank_03_optim_states.pt. +50: [2023-04-29 09:45:17,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_03_optim_states.pt. +43: [2023-04-29 09:45:17,250] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 351 +50: [2023-04-29 09:45:17,250] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 405 +19: [2023-04-29 09:45:17,251] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 +41: [2023-04-29 09:45:17,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_03_optim_states.pt. +41: [2023-04-29 09:45:17,254] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 329 +40: [2023-04-29 09:45:17,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_03_optim_states.pt. +40: [2023-04-29 09:45:17,254] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 327 +15: [2023-04-29 09:45:17,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-04-29 09:45:17,255] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 +32: [2023-04-29 09:45:17,255] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 263 +28: [2023-04-29 09:45:17,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-04-29 09:45:17,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 +58: [2023-04-29 09:45:17,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_03_optim_states.pt. +43: [2023-04-29 09:45:17,256] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 351 + 6: [2023-04-29 09:45:17,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. +58: [2023-04-29 09:45:17,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 469 + 6: [2023-04-29 09:45:17,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 + 9: [2023-04-29 09:45:17,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. +54: [2023-04-29 09:45:17,257] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 437 + 9: [2023-04-29 09:45:17,257] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 +56: [2023-04-29 09:45:17,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_03_optim_states.pt. +56: [2023-04-29 09:45:17,258] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 453 +56: [2023-04-29 09:45:17,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_97_mp_rank_02_optim_states.pt. +56: [2023-04-29 09:45:17,259] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 450 +50: [2023-04-29 09:45:17,259] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 405 +56: [2023-04-29 09:45:17,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_03_optim_states.pt. +56: [2023-04-29 09:45:17,260] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 449 +40: [2023-04-29 09:45:17,261] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 327 +50: [2023-04-29 09:45:17,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_03_optim_states.pt. +41: [2023-04-29 09:45:17,261] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 329 +50: [2023-04-29 09:45:17,261] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 407 + 2: [2023-04-29 09:45:17,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. +15: [2023-04-29 09:45:17,262] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 + 2: [2023-04-29 09:45:17,262] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 + 6: [2023-04-29 09:45:17,263] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 + 9: [2023-04-29 09:45:17,263] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +58: [2023-04-29 09:45:17,264] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 469 +56: [2023-04-29 09:45:17,265] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 450 +27: [2023-04-29 09:45:17,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-04-29 09:45:17,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 +56: [2023-04-29 09:45:17,266] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 453 +28: [2023-04-29 09:45:17,266] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 +23: [2023-04-29 09:45:17,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-04-29 09:45:17,267] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 +56: [2023-04-29 09:45:17,268] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 449 +50: [2023-04-29 09:45:17,268] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 407 +25: [2023-04-29 09:45:17,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. +25: [2023-04-29 09:45:17,269] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 + 2: [2023-04-29 09:45:17,269] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 +26: [2023-04-29 09:45:17,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-04-29 09:45:17,271] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +40: [2023-04-29 09:45:17,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_33_mp_rank_03_optim_states.pt. +40: [2023-04-29 09:45:17,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 323 +62: [2023-04-29 09:45:17,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_03_optim_states.pt. +62: [2023-04-29 09:45:17,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 503 +17: [2023-04-29 09:45:17,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +17: [2023-04-29 09:45:17,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +27: [2023-04-29 09:45:17,273] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 +25: [2023-04-29 09:45:17,275] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 +31: [2023-04-29 09:45:17,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-04-29 09:45:17,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 + 3: [2023-04-29 09:45:17,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. +23: [2023-04-29 09:45:17,276] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 + 3: [2023-04-29 09:45:17,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 +27: [2023-04-29 09:45:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-04-29 09:45:17,278] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +26: [2023-04-29 09:45:17,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +40: [2023-04-29 09:45:17,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 323 +17: [2023-04-29 09:45:17,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +62: [2023-04-29 09:45:17,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 503 +55: [2023-04-29 09:45:17,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_03_optim_states.pt. +55: [2023-04-29 09:45:17,282] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 441 +31: [2023-04-29 09:45:17,282] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 + 5: [2023-04-29 09:45:17,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-04-29 09:45:17,284] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 +27: [2023-04-29 09:45:17,285] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 + 3: [2023-04-29 09:45:17,285] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 + 7: [2023-04-29 09:45:17,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. +48: [2023-04-29 09:45:17,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_03_optim_states.pt. + 7: [2023-04-29 09:45:17,285] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 +48: [2023-04-29 09:45:17,286] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 385 +22: [2023-04-29 09:45:17,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +22: [2023-04-29 09:45:17,287] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 +30: [2023-04-29 09:45:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-04-29 09:45:17,288] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +55: [2023-04-29 09:45:17,289] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 441 +26: [2023-04-29 09:45:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-04-29 09:45:17,289] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 +54: [2023-04-29 09:45:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_89_mp_rank_03_optim_states.pt. +54: [2023-04-29 09:45:17,289] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 435 + 8: [2023-04-29 09:45:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. + 8: [2023-04-29 09:45:17,290] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 + 5: [2023-04-29 09:45:17,291] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 + 9: [2023-04-29 09:45:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. + 9: [2023-04-29 09:45:17,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +56: [2023-04-29 09:45:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_98_mp_rank_02_optim_states.pt. +56: [2023-04-29 09:45:17,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 452 +25: [2023-04-29 09:45:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +25: [2023-04-29 09:45:17,293] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 +22: [2023-04-29 09:45:17,294] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 + 7: [2023-04-29 09:45:17,294] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 +19: [2023-04-29 09:45:17,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +26: [2023-04-29 09:45:17,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +19: [2023-04-29 09:45:17,296] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 +30: [2023-04-29 09:45:17,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +54: [2023-04-29 09:45:17,297] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 435 +40: [2023-04-29 09:45:17,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_34_mp_rank_03_optim_states.pt. +40: [2023-04-29 09:45:17,297] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 325 +62: [2023-04-29 09:45:17,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_122_mp_rank_03_optim_states.pt. +62: [2023-04-29 09:45:17,298] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 501 + 9: [2023-04-29 09:45:17,299] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 + 8: [2023-04-29 09:45:17,299] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 +56: [2023-04-29 09:45:17,300] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 452 +48: [2023-04-29 09:45:17,299] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 385 +25: [2023-04-29 09:45:17,300] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +27: [2023-04-29 09:45:17,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-04-29 09:45:17,301] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 +19: [2023-04-29 09:45:17,302] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +51: [2023-04-29 09:45:17,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_03_optim_states.pt. +51: [2023-04-29 09:45:17,303] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 413 +15: [2023-04-29 09:45:17,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-04-29 09:45:17,304] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 +40: [2023-04-29 09:45:17,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_32_mp_rank_03_optim_states.pt. +40: [2023-04-29 09:45:17,304] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 321 +40: [2023-04-29 09:45:17,306] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 325 +62: [2023-04-29 09:45:17,307] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 501 + 2: [2023-04-29 09:45:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-04-29 09:45:17,308] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 +19: [2023-04-29 09:45:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-04-29 09:45:17,308] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +37: [2023-04-29 09:45:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_03_optim_states.pt. +37: [2023-04-29 09:45:17,309] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 299 +27: [2023-04-29 09:45:17,310] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +40: [2023-04-29 09:45:17,311] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 321 +44: [2023-04-29 09:45:17,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_03_optim_states.pt. +44: [2023-04-29 09:45:17,312] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 359 +51: [2023-04-29 09:45:17,312] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 413 +15: [2023-04-29 09:45:17,313] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +56: [2023-04-29 09:45:17,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_99_mp_rank_03_optim_states.pt. +19: [2023-04-29 09:45:17,315] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 +56: [2023-04-29 09:45:17,315] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 455 +15: [2023-04-29 09:45:17,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-04-29 09:45:17,316] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 +37: [2023-04-29 09:45:17,317] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 299 + 2: [2023-04-29 09:45:17,317] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 +44: [2023-04-29 09:45:17,319] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 359 +31: [2023-04-29 09:45:17,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-04-29 09:45:17,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 + 7: [2023-04-29 09:45:17,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-04-29 09:45:17,322] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 +18: [2023-04-29 09:45:17,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-04-29 09:45:17,322] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +56: [2023-04-29 09:45:17,323] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 455 +15: [2023-04-29 09:45:17,324] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 +25: [2023-04-29 09:45:17,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-04-29 09:45:17,326] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 +25: [2023-04-29 09:45:17,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-04-29 09:45:17,327] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +31: [2023-04-29 09:45:17,328] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 +18: [2023-04-29 09:45:17,329] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 + 7: [2023-04-29 09:45:17,331] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 +15: [2023-04-29 09:45:17,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +46: [2023-04-29 09:45:17,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_57_mp_rank_03_optim_states.pt. +25: [2023-04-29 09:45:17,333] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 +29: [2023-04-29 09:45:17,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +46: [2023-04-29 09:45:17,334] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 371 +29: [2023-04-29 09:45:17,334] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 +15: [2023-04-29 09:45:17,333] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 + 9: [2023-04-29 09:45:17,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 9: [2023-04-29 09:45:17,335] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 +43: [2023-04-29 09:45:17,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_45_mp_rank_03_optim_states.pt. +43: [2023-04-29 09:45:17,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_46_mp_rank_03_optim_states.pt. +25: [2023-04-29 09:45:17,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-04-29 09:45:17,335] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 +43: [2023-04-29 09:45:17,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_44_mp_rank_03_optim_states.pt. +25: [2023-04-29 09:45:17,336] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 +43: [2023-04-29 09:45:17,336] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 347 +43: [2023-04-29 09:45:17,336] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 349 +43: [2023-04-29 09:45:17,336] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 345 +61: [2023-04-29 09:45:17,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_03_optim_states.pt. +61: [2023-04-29 09:45:17,337] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 495 +16: [2023-04-29 09:45:17,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +16: [2023-04-29 09:45:17,340] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 +18: [2023-04-29 09:45:17,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-04-29 09:45:17,341] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 +46: [2023-04-29 09:45:17,341] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 371 +35: [2023-04-29 09:45:17,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_03_optim_states.pt. +30: [2023-04-29 09:45:17,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +35: [2023-04-29 09:45:17,341] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 285 +30: [2023-04-29 09:45:17,341] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 +29: [2023-04-29 09:45:17,341] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 +44: [2023-04-29 09:45:17,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_49_mp_rank_03_optim_states.pt. +15: [2023-04-29 09:45:17,341] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 +44: [2023-04-29 09:45:17,342] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 355 + 9: [2023-04-29 09:45:17,342] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 +25: [2023-04-29 09:45:17,343] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 +59: [2023-04-29 09:45:17,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_03_optim_states.pt. +59: [2023-04-29 09:45:17,343] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 475 +55: [2023-04-29 09:45:17,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_95_mp_rank_03_optim_states.pt. +61: [2023-04-29 09:45:17,344] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 495 +55: [2023-04-29 09:45:17,344] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 447 +43: [2023-04-29 09:45:17,344] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 349 +43: [2023-04-29 09:45:17,345] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 345 +43: [2023-04-29 09:45:17,345] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 347 +47: [2023-04-29 09:45:17,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_61_mp_rank_03_optim_states.pt. +47: [2023-04-29 09:45:17,347] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 379 +16: [2023-04-29 09:45:17,347] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 +28: [2023-04-29 09:45:17,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-04-29 09:45:17,347] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 +18: [2023-04-29 09:45:17,348] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +44: [2023-04-29 09:45:17,349] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 355 +35: [2023-04-29 09:45:17,349] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 285 + 2: [2023-04-29 09:45:17,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. + 2: [2023-04-29 09:45:17,350] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 +31: [2023-04-29 09:45:17,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-04-29 09:45:17,351] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +54: [2023-04-29 09:45:17,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_91_mp_rank_03_optim_states.pt. +54: [2023-04-29 09:45:17,351] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 439 +59: [2023-04-29 09:45:17,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 475 +55: [2023-04-29 09:45:17,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 447 +17: [2023-04-29 09:45:17,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +30: [2023-04-29 09:45:17,352] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 +17: [2023-04-29 09:45:17,352] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +44: [2023-04-29 09:45:17,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_50_mp_rank_03_optim_states.pt. +27: [2023-04-29 09:45:17,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-04-29 09:45:17,352] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +44: [2023-04-29 09:45:17,352] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 357 +47: [2023-04-29 09:45:17,354] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 379 +28: [2023-04-29 09:45:17,356] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 +17: [2023-04-29 09:45:17,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-04-29 09:45:17,357] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +31: [2023-04-29 09:45:17,358] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 + 2: [2023-04-29 09:45:17,358] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 +54: [2023-04-29 09:45:17,359] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 439 +45: [2023-04-29 09:45:17,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_55_mp_rank_03_optim_states.pt. +45: [2023-04-29 09:45:17,359] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 367 +44: [2023-04-29 09:45:17,359] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 357 +17: [2023-04-29 09:45:17,359] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 +27: [2023-04-29 09:45:17,362] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +17: [2023-04-29 09:45:17,364] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 + 4: [2023-04-29 09:45:17,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. +34: [2023-04-29 09:45:17,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_10_mp_rank_03_optim_states.pt. + 4: [2023-04-29 09:45:17,365] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 +34: [2023-04-29 09:45:17,365] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 277 +34: [2023-04-29 09:45:17,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_9_mp_rank_03_optim_states.pt. +34: [2023-04-29 09:45:17,366] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 275 +63: [2023-04-29 09:45:17,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_124_mp_rank_03_optim_states.pt. +63: [2023-04-29 09:45:17,367] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 505 +18: [2023-04-29 09:45:17,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-04-29 09:45:17,368] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +45: [2023-04-29 09:45:17,368] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 367 + 0: [2023-04-29 09:45:17,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-04-29 09:45:17,369] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 +45: [2023-04-29 09:45:17,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_54_mp_rank_03_optim_states.pt. +45: [2023-04-29 09:45:17,371] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 365 +63: [2023-04-29 09:45:17,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_126_mp_rank_03_optim_states.pt. +63: [2023-04-29 09:45:17,371] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 509 +46: [2023-04-29 09:45:17,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_56_mp_rank_03_optim_states.pt. +46: [2023-04-29 09:45:17,372] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 369 +34: [2023-04-29 09:45:17,372] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 277 +34: [2023-04-29 09:45:17,373] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 275 +55: [2023-04-29 09:45:17,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_93_mp_rank_03_optim_states.pt. +55: [2023-04-29 09:45:17,373] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 443 +63: [2023-04-29 09:45:17,373] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 505 + 4: [2023-04-29 09:45:17,375] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 +18: [2023-04-29 09:45:17,375] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 +31: [2023-04-29 09:45:17,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-04-29 09:45:17,376] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 + 1: [2023-04-29 09:45:17,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-04-29 09:45:17,376] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 + 0: [2023-04-29 09:45:17,376] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 +26: [2023-04-29 09:45:17,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-04-29 09:45:17,378] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 +47: [2023-04-29 09:45:17,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_60_mp_rank_03_optim_states.pt. +63: [2023-04-29 09:45:17,378] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 509 +47: [2023-04-29 09:45:17,378] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 377 +46: [2023-04-29 09:45:17,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 369 +45: [2023-04-29 09:45:17,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 365 +60: [2023-04-29 09:45:17,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_112_mp_rank_03_optim_states.pt. +60: [2023-04-29 09:45:17,381] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 481 +55: [2023-04-29 09:45:17,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 443 +31: [2023-04-29 09:45:17,383] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 + 1: [2023-04-29 09:45:17,383] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 +34: [2023-04-29 09:45:17,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_8_mp_rank_03_optim_states.pt. +34: [2023-04-29 09:45:17,384] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 273 +26: [2023-04-29 09:45:17,385] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +47: [2023-04-29 09:45:17,385] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 377 + 3: [2023-04-29 09:45:17,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. +60: [2023-04-29 09:45:17,388] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 481 +44: [2023-04-29 09:45:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_48_mp_rank_03_optim_states.pt. +16: [2023-04-29 09:45:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +44: [2023-04-29 09:45:17,389] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 353 +16: [2023-04-29 09:45:17,390] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 + 3: [2023-04-29 09:45:17,389] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 +34: [2023-04-29 09:45:17,391] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 273 + 7: [2023-04-29 09:45:17,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-04-29 09:45:17,393] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 +29: [2023-04-29 09:45:17,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-04-29 09:45:17,393] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 +37: [2023-04-29 09:45:17,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_20_mp_rank_03_optim_states.pt. +44: [2023-04-29 09:45:17,396] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 353 +37: [2023-04-29 09:45:17,396] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 297 +33: [2023-04-29 09:45:17,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_7_mp_rank_03_optim_states.pt. +16: [2023-04-29 09:45:17,398] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +33: [2023-04-29 09:45:17,398] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 271 +29: [2023-04-29 09:45:17,400] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 +19: [2023-04-29 09:45:17,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. +19: [2023-04-29 09:45:17,402] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 + 7: [2023-04-29 09:45:17,402] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 +37: [2023-04-29 09:45:17,403] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 297 +33: [2023-04-29 09:45:17,407] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 271 +30: [2023-04-29 09:45:17,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-04-29 09:45:17,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 +16: [2023-04-29 09:45:17,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +16: [2023-04-29 09:45:17,410] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +19: [2023-04-29 09:45:17,410] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 +34: [2023-04-29 09:45:17,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_11_mp_rank_03_optim_states.pt. +35: [2023-04-29 09:45:17,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_03_optim_states.pt. +34: [2023-04-29 09:45:17,410] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 279 +35: [2023-04-29 09:45:17,410] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 281 +16: [2023-04-29 09:45:17,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-04-29 09:45:17,413] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 + 3: [2023-04-29 09:45:17,412] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 +37: [2023-04-29 09:45:17,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_23_mp_rank_03_optim_states.pt. +30: [2023-04-29 09:45:17,416] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 +37: [2023-04-29 09:45:17,416] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 303 +34: [2023-04-29 09:45:17,417] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 279 +57: [2023-04-29 09:45:17,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_103_mp_rank_03_optim_states.pt. +16: [2023-04-29 09:45:17,418] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 +57: [2023-04-29 09:45:17,418] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 463 +35: [2023-04-29 09:45:17,419] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 281 +59: [2023-04-29 09:45:17,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_108_mp_rank_03_optim_states.pt. +59: [2023-04-29 09:45:17,419] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 473 +23: [2023-04-29 09:45:17,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-04-29 09:45:17,420] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 +19: [2023-04-29 09:45:17,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-04-29 09:45:17,420] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +16: [2023-04-29 09:45:17,421] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 +37: [2023-04-29 09:45:17,423] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 303 +46: [2023-04-29 09:45:17,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_58_mp_rank_03_optim_states.pt. +57: [2023-04-29 09:45:17,425] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 463 +46: [2023-04-29 09:45:17,425] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 373 +47: [2023-04-29 09:45:17,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_62_mp_rank_03_optim_states.pt. +47: [2023-04-29 09:45:17,425] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 381 +23: [2023-04-29 09:45:17,426] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +63: [2023-04-29 09:45:17,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_127_mp_rank_03_optim_states.pt. +63: [2023-04-29 09:45:17,426] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 511 +59: [2023-04-29 09:45:17,427] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 473 +12: [2023-04-29 09:45:17,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-04-29 09:45:17,429] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 +59: [2023-04-29 09:45:17,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_111_mp_rank_03_optim_states.pt. +59: [2023-04-29 09:45:17,429] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 479 +19: [2023-04-29 09:45:17,430] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 +59: [2023-04-29 09:45:17,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_110_mp_rank_03_optim_states.pt. +59: [2023-04-29 09:45:17,430] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 477 +63: [2023-04-29 09:45:17,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_125_mp_rank_03_optim_states.pt. +63: [2023-04-29 09:45:17,433] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 507 +46: [2023-04-29 09:45:17,433] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 373 +47: [2023-04-29 09:45:17,434] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 381 +63: [2023-04-29 09:45:17,434] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 511 + 5: [2023-04-29 09:45:17,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. + 5: [2023-04-29 09:45:17,435] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 +12: [2023-04-29 09:45:17,436] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +46: [2023-04-29 09:45:17,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_59_mp_rank_03_optim_states.pt. +46: [2023-04-29 09:45:17,436] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 375 +24: [2023-04-29 09:45:17,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-04-29 09:45:17,437] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 +59: [2023-04-29 09:45:17,438] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 479 +59: [2023-04-29 09:45:17,438] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 477 +63: [2023-04-29 09:45:17,441] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 507 +46: [2023-04-29 09:45:17,443] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 375 + 5: [2023-04-29 09:45:17,443] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 +57: [2023-04-29 09:45:17,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_100_mp_rank_03_optim_states.pt. +57: [2023-04-29 09:45:17,446] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 457 + 7: [2023-04-29 09:45:17,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. +26: [2023-04-29 09:45:17,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-04-29 09:45:17,448] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 + 7: [2023-04-29 09:45:17,448] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 +60: [2023-04-29 09:45:17,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_115_mp_rank_03_optim_states.pt. +60: [2023-04-29 09:45:17,449] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 487 +24: [2023-04-29 09:45:17,449] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 +57: [2023-04-29 09:45:17,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 457 +38: [2023-04-29 09:45:17,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_24_mp_rank_03_optim_states.pt. +38: [2023-04-29 09:45:17,456] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 305 +49: [2023-04-29 09:45:17,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_68_mp_rank_03_optim_states.pt. +26: [2023-04-29 09:45:17,456] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 +49: [2023-04-29 09:45:17,456] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 393 +61: [2023-04-29 09:45:17,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_03_optim_states.pt. +61: [2023-04-29 09:45:17,456] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 493 + 7: [2023-04-29 09:45:17,456] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 +60: [2023-04-29 09:45:17,457] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 487 +45: [2023-04-29 09:45:17,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_53_mp_rank_03_optim_states.pt. +45: [2023-04-29 09:45:17,458] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 363 + 0: [2023-04-29 09:45:17,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-04-29 09:45:17,460] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 +38: [2023-04-29 09:45:17,463] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 305 +61: [2023-04-29 09:45:17,464] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 493 +49: [2023-04-29 09:45:17,464] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 393 + 0: [2023-04-29 09:45:17,467] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 +45: [2023-04-29 09:45:17,467] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 363 +29: [2023-04-29 09:45:17,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-04-29 09:45:17,470] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 + 3: [2023-04-29 09:45:17,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-04-29 09:45:17,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 +29: [2023-04-29 09:45:17,477] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +55: [2023-04-29 09:45:17,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_94_mp_rank_03_optim_states.pt. +55: [2023-04-29 09:45:17,478] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 445 +61: [2023-04-29 09:45:17,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_117_mp_rank_03_optim_states.pt. +61: [2023-04-29 09:45:17,479] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 491 +61: [2023-04-29 09:45:17,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_116_mp_rank_03_optim_states.pt. +61: [2023-04-29 09:45:17,480] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 489 +45: [2023-04-29 09:45:17,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_52_mp_rank_03_optim_states.pt. +45: [2023-04-29 09:45:17,480] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 361 +49: [2023-04-29 09:45:17,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_71_mp_rank_03_optim_states.pt. +49: [2023-04-29 09:45:17,484] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 399 +61: [2023-04-29 09:45:17,486] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 491 +55: [2023-04-29 09:45:17,486] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 445 +45: [2023-04-29 09:45:17,487] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 361 +57: [2023-04-29 09:45:17,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_102_mp_rank_03_optim_states.pt. +57: [2023-04-29 09:45:17,488] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 461 +61: [2023-04-29 09:45:17,488] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 489 +18: [2023-04-29 09:45:17,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-04-29 09:45:17,490] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 +49: [2023-04-29 09:45:17,491] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 399 +57: [2023-04-29 09:45:17,496] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 461 +60: [2023-04-29 09:45:17,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_113_mp_rank_03_optim_states.pt. +60: [2023-04-29 09:45:17,496] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 483 +18: [2023-04-29 09:45:17,498] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +47: [2023-04-29 09:45:17,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_63_mp_rank_03_optim_states.pt. +47: [2023-04-29 09:45:17,500] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 383 + 3: [2023-04-29 09:45:17,500] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 + 0: [2023-04-29 09:45:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-04-29 09:45:17,502] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 +49: [2023-04-29 09:45:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_69_mp_rank_03_optim_states.pt. +49: [2023-04-29 09:45:17,503] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 395 +60: [2023-04-29 09:45:17,505] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 483 +47: [2023-04-29 09:45:17,509] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 383 +49: [2023-04-29 09:45:17,510] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 395 + 0: [2023-04-29 09:45:17,510] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 +52: [2023-04-29 09:45:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_83_mp_rank_03_optim_states.pt. +52: [2023-04-29 09:45:17,516] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 423 +57: [2023-04-29 09:45:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_101_mp_rank_03_optim_states.pt. +57: [2023-04-29 09:45:17,517] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 459 +52: [2023-04-29 09:45:17,524] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 423 +57: [2023-04-29 09:45:17,524] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 459 +49: [2023-04-29 09:45:17,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_70_mp_rank_03_optim_states.pt. +49: [2023-04-29 09:45:17,526] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 397 +49: [2023-04-29 09:45:17,533] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 397 + 0: [2023-04-29 09:45:17,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-04-29 09:45:17,536] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 +60: [2023-04-29 09:45:17,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_114_mp_rank_03_optim_states.pt. +60: [2023-04-29 09:45:17,539] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 485 +36: [2023-04-29 09:45:17,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_16_mp_rank_03_optim_states.pt. +36: [2023-04-29 09:45:17,539] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 289 + 0: [2023-04-29 09:45:17,544] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +60: [2023-04-29 09:45:17,546] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 485 +36: [2023-04-29 09:45:17,546] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 289 +52: [2023-04-29 09:45:17,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_82_mp_rank_03_optim_states.pt. +52: [2023-04-29 09:45:17,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 421 +52: [2023-04-29 09:45:17,574] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 421 +36: [2023-04-29 09:45:17,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_18_mp_rank_03_optim_states.pt. +36: [2023-04-29 09:45:17,582] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 293 +36: [2023-04-29 09:45:17,589] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 293 +52: [2023-04-29 09:45:17,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_81_mp_rank_03_optim_states.pt. +52: [2023-04-29 09:45:17,598] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 419 +36: [2023-04-29 09:45:17,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_17_mp_rank_03_optim_states.pt. +52: [2023-04-29 09:45:17,606] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 419 +36: [2023-04-29 09:45:17,607] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 291 +36: [2023-04-29 09:45:17,614] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 291 +52: [2023-04-29 09:45:17,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_80_mp_rank_03_optim_states.pt. +52: [2023-04-29 09:45:17,631] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 417 + 3: [2023-04-29 09:45:17,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 3: [2023-04-29 09:45:17,634] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 +52: [2023-04-29 09:45:17,637] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 417 + 3: [2023-04-29 09:45:17,666] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 +36: [2023-04-29 09:45:17,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b/global_step84877/bf16_zero_pp_rank_19_mp_rank_03_optim_states.pt. +36: [2023-04-29 09:45:17,682] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 295 +36: [2023-04-29 09:45:17,691] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 295 + 0: successfully loaded checkpoint from lm1-8b7-178b-c4-repetitions/8b7178b178b at iteration 0 +63: time (ms) | load-checkpoint: 13982.40 + 0: estimated model parameters: 8.905678848 + 0: estimated model parameters without embeddings: 8.458985472 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-04-29 09:45:18 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 51200 + 0: test: 51200 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.058619 seconds + 0: number of documents: 835726 + 0: > dataset split: + 0: train: + 0: document indices in [0, 835726) total of 835726 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_400M_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_400M_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_400M_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.060 seconds + 0: total number of samples: 195101 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.013980 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_51200ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_51200ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_51200ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.079 seconds + 0: total number of samples: 84978 + 0: total number of epochs: 1 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-04-29 09:45:26 + 0: done with setup ... + 0: training ... +63: time (ms) | model-and-optimizer-setup: 33441.60 | train/valid/test-data-iterators-setup: 7180.78 + 0: [after training is done] datetime: 2023-04-29 09:45:26 +63: ----------------------------------------------------------------------------------------------------------------- +63: validation loss at the end of training for val data | lm loss value: 2.336741E+00 | lm loss PPL: 1.034746E+01 | +63: ----------------------------------------------------------------------------------------------------------------- +END 3430821: Sat 29 Apr 2023 09:47:00 AM EEST